diff options
111 files changed, 19306 insertions, 326 deletions
diff --git a/sys/amd64/amd64/autoconf.c b/sys/amd64/amd64/autoconf.c index 5d297bfa953f..70e0b137dfee 100644 --- a/sys/amd64/amd64/autoconf.c +++ b/sys/amd64/amd64/autoconf.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)autoconf.c 7.1 (Berkeley) 5/9/91 - * $Id$ + * $Id: autoconf.c,v 1.64 1997/02/22 09:32:08 peter Exp $ */ /* @@ -45,6 +45,8 @@ * devices are determined (from possibilities mentioned in ioconf.c), * and the drivers are initialized. */ +#include "opt_smp.h" + #include <sys/param.h> #include <sys/systm.h> #include <sys/buf.h> @@ -61,6 +63,10 @@ #include <machine/md_var.h> #include <i386/isa/icu.h> /* For interrupts */ +#if defined(APIC_IO) +#include <machine/smp.h> +#endif /* APIC_IO */ + #include "isa.h" #if NISA > 0 #include <i386/isa/isa_device.h> @@ -158,8 +164,18 @@ configure(dummy) configure_start(); /* Allow all routines to decide for themselves if they want intrs */ +#if defined(APIC_IO) + configure_local_apic(); + enable_intr(); +#else enable_intr(); INTREN(IRQ_SLAVE); +#endif /* APIC_IO */ + +#if NCRD > 0 + /* Before isa_configure to avoid ISA drivers finding our cards */ + pccard_configure(); +#endif #if NEISA > 0 eisa_configure(); diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S index a74cd09b5a6c..014ddd2f2c57 100644 --- a/sys/amd64/amd64/cpu_switch.S +++ b/sys/amd64/amd64/cpu_switch.S @@ -33,16 +33,24 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: swtch.s,v 1.46 1997/04/20 06:41:26 phk Exp $ + * $Id: swtch.s,v 1.47 1997/04/22 06:55:29 jdp Exp $ */ #include "npx.h" #include "opt_user_ldt.h" +#include "opt_smp.h" +#include "opt_smp_privpages.h" #include <sys/rtprio.h> #include <machine/asmacros.h> #include <machine/spl.h> +#include <machine/smpasm.h> +#include <machine/smptests.h> /** TEST_LOPRIO */ + +#if defined(SMP) && defined(SMP_PRIVPAGES) +#include <machine/pmap.h> +#endif #include "assym.s" @@ -61,8 +69,12 @@ * queues. */ .data +#ifndef SMP .globl _curpcb _curpcb: .long 0 /* pointer to curproc's PCB area */ +#endif + .globl _whichqs, _whichrtqs, _whichidqs + _whichqs: .long 0 /* which run queues have data */ _whichrtqs: .long 0 /* which realtime run queues have data */ _whichidqs: .long 0 /* which idletime run queues have data */ @@ -229,16 +241,30 @@ rem3id: .asciz "remrq.id" /* * When no processes are on the runq, cpu_switch() branches to _idle * to wait for something to come ready. + * + * NOTE: on an SMP system this routine is a startup-only code path. + * once initialization is over, meaning the idle procs have been + * created, we should NEVER branch here. */ ALIGN_TEXT _idle: +#ifdef SMP + movl _smp_active, %eax + cmpl $0, %eax + jnz badsw +#endif /* SMP */ xorl %ebp,%ebp movl $HIDENAME(tmpstk),%esp movl _IdlePTD,%ecx movl %ecx,%cr3 /* update common_tss.tss_esp0 pointer */ +#ifdef SMP + GETCPUID(%eax) + movl _SMPcommon_tss_ptr(,%eax,4), %eax +#else movl $_common_tss, %eax +#endif movl %esp, TSS_ESP0(%eax) #ifdef TSS_IS_CACHED /* example only */ @@ -288,13 +314,20 @@ ENTRY(default_halt) * cpu_switch() */ ENTRY(cpu_switch) + /* switch to new process. first, save context as needed */ - movl _curproc,%ecx + GETCURPROC(%ecx) /* if no process to save, don't bother */ testl %ecx,%ecx je sw1 +#ifdef SMP + movb P_ONCPU(%ecx), %al /* save "last" cpu */ + movb %al, P_LASTCPU(%ecx) + movb $0xff, P_ONCPU(%ecx) /* "leave" the cpu */ +#endif + movl P_ADDR(%ecx),%ecx movl (%esp),%eax /* Hardware registers */ @@ -305,10 +338,19 @@ ENTRY(cpu_switch) movl %esi,PCB_ESI(%ecx) movl %edi,PCB_EDI(%ecx) +#ifdef SMP + movl _mp_lock, %eax + cmpl $0xffffffff, %eax /* is it free? */ + je badsw /* yes, bad medicine! */ + andl $0x00ffffff, %eax /* clear CPU portion */ + movl %eax,PCB_MPNEST(%ecx) /* store it */ +#endif /* SMP */ + #if NNPX > 0 /* have we used fp, and need a save? */ - mov _curproc,%eax - cmp %eax,_npxproc + GETCURPROC(%eax) + GETNPXPROC(%ebx) + cmp %eax,%ebx jne 1f addl $PCB_SAVEFPU,%ecx /* h/w bugs make saving complicated */ pushl %ecx @@ -319,7 +361,7 @@ ENTRY(cpu_switch) movb $1,_intr_nesting_level /* charge Intr, not Sys/Idle */ - movl $0,_curproc /* out of process */ + SETCURPROC($0, %edi) /* save is done, now choose a new process or idle */ sw1: @@ -417,9 +459,30 @@ swtch_com: movl P_ADDR(%ecx),%edx movl PCB_CR3(%edx),%ebx +#if defined(SMP) && defined(SMP_PRIVPAGES) + /* Grab the private PT pointer from the outgoing process's PTD */ + movl $_PTD,%esi + movl 4*MPPTDI(%esi), %eax /* fetch cpu's prv pt */ +#endif + /* switch address space */ movl %ebx,%cr3 +#if defined(SMP) && defined(SMP_PRIVPAGES) + /* Copy the private PT to the new process's PTD */ + /* XXX yuck, the _PTD changes when we switch, so we have to + * reload %cr3 after changing the address space. + * We need to fix this by storing a pointer to the virtual + * location of the per-process PTD in the PCB or something quick. + * Dereferencing proc->vm_map->pmap->p_pdir[] is painful in asm. + */ + movl $_PTD,%esi + movl %eax, 4*MPPTDI(%esi) /* restore cpu's prv page */ + + /* XXX: we have just changed the page tables.. reload.. */ + movl %ebx,%cr3 +#endif + #ifdef HOW_TO_SWITCH_TSS /* example only */ /* Fix up tss pointer to floating pcb/stack structure */ /* XXX probably lots faster to store the 64 bits of tss entry @@ -435,7 +498,12 @@ swtch_com: #endif /* update common_tss.tss_esp0 pointer */ +#ifdef SMP + GETCPUID(%eax) + movl _SMPcommon_tss_ptr(,%eax,4), %eax +#else movl $_common_tss, %eax +#endif movl %edx, %ebx /* pcb */ addl $(UPAGES * PAGE_SIZE), %ebx movl %ebx, TSS_ESP0(%eax) @@ -457,9 +525,32 @@ swtch_com: movl PCB_EIP(%edx),%eax movl %eax,(%esp) - movl %edx,_curpcb - movl %ecx,_curproc /* into next process */ +#ifdef SMP + GETCPUID(%eax) + movb %al, P_ONCPU(%ecx) +#endif + SETCURPCB(%edx, %eax) + SETCURPROC(%ecx, %eax) + movb $0,_intr_nesting_level +#ifdef SMP + movl _apic_base, %eax /* base addr of LOCAL APIC */ +#if defined(TEST_LOPRIO) + /** + * FIXME: the belief here is that we ALWAYS leave here + * holding a the lock, is this TRUE??? + */ + pushl %edx + movl APIC_TPR(%eax), %edx /* get TPR register contents */ + andl $~0xff, %edx /* clear the prio field */ + movl %edx, APIC_TPR(%eax) /* now hold loprio for INTs */ + popl %edx +#endif /* TEST_LOPRIO */ + movl APIC_ID(%eax), %eax /* APIC ID register */ + andl $APIC_ID_MASK, %eax /* extract ID portion */ + orl PCB_MPNEST(%edx), %eax /* add count from PROC */ + movl %eax, _mp_lock /* load the mp_lock */ +#endif /* SMP */ #ifdef USER_LDT cmpl $0, PCB_USERLDT(%edx) @@ -520,7 +611,7 @@ ENTRY(savectx) * have to handle h/w bugs for reloading. We used to lose the * parent's npx state for forks by forgetting to reload. */ - mov _npxproc,%eax + GETNPXPROC(%eax) testl %eax,%eax je 1f diff --git a/sys/amd64/amd64/db_interface.c b/sys/amd64/amd64/db_interface.c index c286027d53bd..adee9ff7fe47 100644 --- a/sys/amd64/amd64/db_interface.c +++ b/sys/amd64/amd64/db_interface.c @@ -23,7 +23,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: db_interface.c,v 1.26 1997/02/22 09:32:14 peter Exp $ + * $Id: db_interface.c,v 1.27 1997/03/28 12:56:07 bde Exp $ */ /* @@ -265,6 +265,8 @@ Debugger(msg) if (!in_Debugger) { in_Debugger = 1; + printf("CR0=%08x",rcr0()); + printf("CR3=%08x",rcr3()); db_printf("Debugger(\"%s\")\n", msg); breakpoint(); in_Debugger = 0; diff --git a/sys/amd64/amd64/db_trace.c b/sys/amd64/amd64/db_trace.c index 546df763b208..f8a934cf05ed 100644 --- a/sys/amd64/amd64/db_trace.c +++ b/sys/amd64/amd64/db_trace.c @@ -23,7 +23,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: db_trace.c,v 1.19 1997/02/28 12:36:18 bde Exp $ + * $Id: db_trace.c,v 1.20 1997/03/01 05:44:09 ache Exp $ */ #include <sys/param.h> @@ -199,6 +199,8 @@ db_nextframe(fp, ip) db_printf( "--- trap %#n, eip = %#n, esp = %#n, ebp = %#n ---\n", tf->tf_trapno, eip, esp, ebp); + db_printf("--- curproc = 0x%x, pid = %d ---\n", + curproc, curproc ? curproc->p_pid : 0); } break; case SYSCALL: @@ -208,6 +210,8 @@ db_nextframe(fp, ip) db_printf( "--- syscall %#n, eip = %#n, esp = %#n, ebp = %#n ---\n", tf->tf_eax, eip, esp, ebp); + db_printf("--- curproc = 0x%x, pid = %d ---\n", + curproc, curproc ? curproc->p_pid : 0); } break; case INTERRUPT: @@ -218,6 +222,8 @@ db_nextframe(fp, ip) db_printf( "--- interrupt, eip = %#n, esp = %#n, ebp = %#n ---\n", eip, esp, ebp); + db_printf("--- curproc = 0x%x, pid = %d ---\n", + curproc, curproc ? curproc->p_pid : 0); } break; default: diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S index 0ab3b33952e2..2705c08efcfe 100644 --- a/sys/amd64/amd64/exception.S +++ b/sys/amd64/amd64/exception.S @@ -30,9 +30,10 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: exception.s,v 1.23 1997/04/07 11:42:09 peter Exp $ + * $Id: exception.s,v 1.24 1997/04/13 16:58:08 bde Exp $ */ +#include "opt_smp.h" #include "npx.h" /* NNPX */ #include "assym.s" /* system defines */ #include <sys/errno.h> /* error return codes */ @@ -42,6 +43,7 @@ #include <sys/syscall.h> /* syscall numbers */ #include <machine/asmacros.h> /* miscellaneous macros */ #include <sys/cdefs.h> /* CPP macros */ +#include <machine/smpasm.h> #define KDSEL 0x10 /* kernel data selector */ #define SEL_RPL_MASK 0x0003 @@ -138,6 +140,9 @@ IDTVEC(fpu) movl _cpl,%eax pushl %eax pushl $0 /* dummy unit to finish building intr frame */ +#ifdef SMP + call _get_mplock +#endif /* SMP */ incl _cnt+V_TRAP orl $SWI_AST_MASK,%eax movl %eax,_cpl @@ -163,6 +168,9 @@ alltraps_with_regs_pushed: movl %ax,%es FAKE_MCOUNT(12*4(%esp)) calltrap: +#ifdef SMP + call _get_mplock +#endif /* SMP */ FAKE_MCOUNT(_btrap) /* init "from" _btrap -> calltrap */ incl _cnt+V_TRAP orl $SWI_AST_MASK,_cpl @@ -212,6 +220,9 @@ IDTVEC(syscall) movl %eax,TF_EFLAGS(%esp) movl $7,TF_ERR(%esp) /* sizeof "lcall 7,0" */ FAKE_MCOUNT(12*4(%esp)) +#ifdef SMP + call _get_mplock +#endif /* SMP */ incl _cnt+V_SYSCALL movl $SWI_AST_MASK,_cpl call _syscall @@ -238,6 +249,9 @@ IDTVEC(int0x80_syscall) movl %ax,%es movl $2,TF_ERR(%esp) /* sizeof "int 0x80" */ FAKE_MCOUNT(12*4(%esp)) +#ifdef SMP + call _get_mplock +#endif /* SMP */ incl _cnt+V_SYSCALL movl $SWI_AST_MASK,_cpl call _syscall @@ -254,7 +268,13 @@ ENTRY(fork_trampoline) movl $SWI_AST_MASK,_cpl call _splz +#if defined(SMP) + GETCPUID(%eax) + leal _SMPruntime(,%eax,8), %eax + pushl %eax +#else pushl $_runtime +#endif call _microtime popl %eax diff --git a/sys/amd64/amd64/exception.s b/sys/amd64/amd64/exception.s index 0ab3b33952e2..2705c08efcfe 100644 --- a/sys/amd64/amd64/exception.s +++ b/sys/amd64/amd64/exception.s @@ -30,9 +30,10 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: exception.s,v 1.23 1997/04/07 11:42:09 peter Exp $ + * $Id: exception.s,v 1.24 1997/04/13 16:58:08 bde Exp $ */ +#include "opt_smp.h" #include "npx.h" /* NNPX */ #include "assym.s" /* system defines */ #include <sys/errno.h> /* error return codes */ @@ -42,6 +43,7 @@ #include <sys/syscall.h> /* syscall numbers */ #include <machine/asmacros.h> /* miscellaneous macros */ #include <sys/cdefs.h> /* CPP macros */ +#include <machine/smpasm.h> #define KDSEL 0x10 /* kernel data selector */ #define SEL_RPL_MASK 0x0003 @@ -138,6 +140,9 @@ IDTVEC(fpu) movl _cpl,%eax pushl %eax pushl $0 /* dummy unit to finish building intr frame */ +#ifdef SMP + call _get_mplock +#endif /* SMP */ incl _cnt+V_TRAP orl $SWI_AST_MASK,%eax movl %eax,_cpl @@ -163,6 +168,9 @@ alltraps_with_regs_pushed: movl %ax,%es FAKE_MCOUNT(12*4(%esp)) calltrap: +#ifdef SMP + call _get_mplock +#endif /* SMP */ FAKE_MCOUNT(_btrap) /* init "from" _btrap -> calltrap */ incl _cnt+V_TRAP orl $SWI_AST_MASK,_cpl @@ -212,6 +220,9 @@ IDTVEC(syscall) movl %eax,TF_EFLAGS(%esp) movl $7,TF_ERR(%esp) /* sizeof "lcall 7,0" */ FAKE_MCOUNT(12*4(%esp)) +#ifdef SMP + call _get_mplock +#endif /* SMP */ incl _cnt+V_SYSCALL movl $SWI_AST_MASK,_cpl call _syscall @@ -238,6 +249,9 @@ IDTVEC(int0x80_syscall) movl %ax,%es movl $2,TF_ERR(%esp) /* sizeof "int 0x80" */ FAKE_MCOUNT(12*4(%esp)) +#ifdef SMP + call _get_mplock +#endif /* SMP */ incl _cnt+V_SYSCALL movl $SWI_AST_MASK,_cpl call _syscall @@ -254,7 +268,13 @@ ENTRY(fork_trampoline) movl $SWI_AST_MASK,_cpl call _splz +#if defined(SMP) + GETCPUID(%eax) + leal _SMPruntime(,%eax,8), %eax + pushl %eax +#else pushl $_runtime +#endif call _microtime popl %eax diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c index 6e3f5c77e077..402b1cebb76f 100644 --- a/sys/amd64/amd64/fpu.c +++ b/sys/amd64/amd64/fpu.c @@ -32,7 +32,7 @@ * SUCH DAMAGE. * * from: @(#)npx.c 7.2 (Berkeley) 5/12/91 - * $Id: npx.c,v 1.40 1997/03/24 11:23:58 bde Exp $ + * $Id: npx.c,v 1.41 1997/04/22 06:55:38 jdp Exp $ */ #include "npx.h" @@ -40,6 +40,7 @@ #include "opt_cpu.h" #include "opt_math_emulate.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -60,6 +61,10 @@ #include <machine/trap.h> #include <machine/clock.h> #include <machine/specialreg.h> +#if defined(APIC_IO) +#include <machine/apic.h> +#include <machine/mpapic.h> +#endif /* APIC_IO */ #include <i386/isa/icu.h> #include <i386/isa/isa_device.h> @@ -133,7 +138,12 @@ SYSCTL_INT(_hw,HW_FLOATINGPT, floatingpoint, "Floatingpoint instructions executed in hardware"); static u_int npx0_imask = SWI_CLOCK_MASK; +#ifdef SMP +#define npxproc (SMPnpxproc[cpunumber()]) +struct proc *SMPnpxproc[NCPU]; +#else struct proc *npxproc; +#endif static bool_t npx_ex16; static bool_t npx_exists; @@ -148,8 +158,28 @@ static volatile u_int npx_traps_while_probing; * interrupts. We'll still need a special exception 16 handler. The busy * latch stuff in probeintr() can be moved to npxprobe(). */ - inthand_t probeintr; + +#if defined(APIC_IO) + +asm +(" + .text + .p2align 2,0x90 +" __XSTRING(CNAME(probeintr)) ": + ss + incl " __XSTRING(CNAME(npx_intrs_while_probing)) " + pushl %eax + movl " __XSTRING(CNAME(apic_base)) ",%eax # EOI to local APIC + movl $0,0xb0(,%eax,1) # movl $0, APIC_EOI(%eax) + movb $0,%al + outb %al,$0xf0 # clear BUSY# latch + popl %eax + iret +"); + +#else + asm (" .text @@ -167,6 +197,8 @@ asm iret "); +#endif /* APIC_IO */ + inthand_t probetrap; asm (" @@ -191,8 +223,12 @@ npxprobe(dvp) { int result; u_long save_eflags; +#if defined(APIC_IO) + u_int save_apic_mask; +#else u_char save_icu1_mask; u_char save_icu2_mask; +#endif /* APIC_IO */ struct gate_descriptor save_idt_npxintr; struct gate_descriptor save_idt_npxtrap; /* @@ -205,20 +241,32 @@ npxprobe(dvp) npx_intrno = NRSVIDT + ffs(dvp->id_irq) - 1; save_eflags = read_eflags(); disable_intr(); +#if defined(APIC_IO) + save_apic_mask = INTRGET(); +#else save_icu1_mask = inb(IO_ICU1 + 1); save_icu2_mask = inb(IO_ICU2 + 1); +#endif /* APIC_IO */ save_idt_npxintr = idt[npx_intrno]; save_idt_npxtrap = idt[16]; +#if defined(APIC_IO) + INTRSET( ~dvp->id_irq ); +#else outb(IO_ICU1 + 1, ~(IRQ_SLAVE | dvp->id_irq)); outb(IO_ICU2 + 1, ~(dvp->id_irq >> 8)); +#endif /* APIC_IO */ setidt(16, probetrap, SDT_SYS386TGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); setidt(npx_intrno, probeintr, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); npx_idt_probeintr = idt[npx_intrno]; enable_intr(); result = npxprobe1(dvp); disable_intr(); +#if defined(APIC_IO) + INTRSET( save_apic_mask ); +#else outb(IO_ICU1 + 1, save_icu1_mask); outb(IO_ICU2 + 1, save_icu2_mask); +#endif /* APIC_IO */ idt[npx_intrno] = save_idt_npxintr; idt[16] = save_idt_npxtrap; write_eflags(save_eflags); @@ -364,7 +412,8 @@ npxattach(dvp) } npxinit(__INITIAL_NPXCW__); -#ifdef I586_CPU +#if defined(I586_CPU) && !defined(SMP) + /* FPU not working under SMP yet */ if (cpu_class == CPUCLASS_586 && npx_ex16) { if (!(dvp->id_flags & NPX_DISABLE_I586_OPTIMIZED_BCOPY)) { bcopy_vector = i586_bcopy; @@ -576,18 +625,32 @@ void npxsave(addr) struct save87 *addr; { +#if defined(APIC_IO) + u_int apic_mask; + u_int old_apic_mask; +#else u_char icu1_mask; u_char icu2_mask; u_char old_icu1_mask; u_char old_icu2_mask; +#endif /* APIC_IO */ struct gate_descriptor save_idt_npxintr; disable_intr(); +#if defined(APIC_IO) + old_apic_mask = INTRGET(); +#else old_icu1_mask = inb(IO_ICU1 + 1); old_icu2_mask = inb(IO_ICU2 + 1); +#endif /* APIC_IO */ save_idt_npxintr = idt[npx_intrno]; +#if defined(APIC_IO) + /** FIXME: try clrIoApicMaskBit( npx0_imask ); */ + INTRSET( old_apic_mask & ~(npx0_imask & 0xffff) ); +#else outb(IO_ICU1 + 1, old_icu1_mask & ~(IRQ_SLAVE | npx0_imask)); outb(IO_ICU2 + 1, old_icu2_mask & ~(npx0_imask >> 8)); +#endif /* APIC_IO */ idt[npx_intrno] = npx_idt_probeintr; enable_intr(); stop_emulating(); @@ -596,6 +659,11 @@ npxsave(addr) start_emulating(); npxproc = NULL; disable_intr(); +#if defined(APIC_IO) + apic_mask = INTRGET(); /* masks may have changed */ + INTRSET( (apic_mask & ~(npx0_imask & 0xffff)) | + (old_apic_mask & (npx0_imask & 0xffff))); +#else icu1_mask = inb(IO_ICU1 + 1); /* masks may have changed */ icu2_mask = inb(IO_ICU2 + 1); outb(IO_ICU1 + 1, @@ -603,6 +671,7 @@ npxsave(addr) outb(IO_ICU2 + 1, (icu2_mask & ~(npx0_imask >> 8)) | (old_icu2_mask & (npx0_imask >> 8))); +#endif /* APIC_IO */ idt[npx_intrno] = save_idt_npxintr; enable_intr(); /* back to usual state */ } diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c index c1d3b4c1b752..e15535b82786 100644 --- a/sys/amd64/amd64/genassym.c +++ b/sys/amd64/amd64/genassym.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)genassym.c 5.11 (Berkeley) 5/10/91 - * $Id: genassym.c,v 1.43 1997/04/07 06:45:11 peter Exp $ + * $Id: genassym.c,v 1.44 1997/04/07 07:15:48 peter Exp $ */ #include <stdio.h> @@ -102,6 +102,8 @@ main() printf("#define\tP_WCHAN %p\n", &p->p_wchan); printf("#define\tP_FLAG %p\n", &p->p_flag); printf("#define\tP_PID %p\n", &p->p_pid); + printf("#define\tP_ONCPU %p\n", &p->p_oncpu); + printf("#define\tP_LASTCPU %p\n", &p->p_lastcpu); printf("#define\tSSLEEP %d\n", SSLEEP); printf("#define\tSRUN %d\n", SRUN); printf("#define\tV_TRAP %p\n", &vm->v_trap); @@ -131,6 +133,7 @@ main() printf("#define\tPCB_EIP %p\n", &pcb->pcb_eip); printf("#define\tTSS_ESP0 %p\n", &tss->tss_esp0); printf("#define\tPCB_USERLDT %p\n", &pcb->pcb_ldt); + printf("#define\tPCB_MPNEST %p\n", &pcb->pcb_mpnest); printf("#define\tU_PROF %p\n", &up->u_stats.p_prof); printf("#define\tU_PROFSCALE %p\n", &up->u_stats.p_prof.pr_scale); printf("#define\tPR_BASE %p\n", &uprof->pr_base); diff --git a/sys/amd64/amd64/identcpu.c b/sys/amd64/amd64/identcpu.c index 5defd2aeec00..21a62de7811a 100644 --- a/sys/amd64/amd64/identcpu.c +++ b/sys/amd64/amd64/identcpu.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * from: Id: machdep.c,v 1.193 1996/06/18 01:22:04 bde Exp - * $Id: identcpu.c,v 1.15 1997/04/22 06:55:23 jdp Exp $ + * $Id: identcpu.c,v 1.16 1997/04/26 04:08:45 kato Exp $ */ #include "opt_cpu.h" @@ -307,17 +307,21 @@ printcpuinfo(void) #endif #if defined(I586_CPU) case CPUCLASS_586: +#ifndef SMP printf("%d.%02d-MHz ", (i586_ctr_freq + 4999) / 1000000, ((i586_ctr_freq + 4999) / 10000) % 100); +#endif printf("586"); break; #endif #if defined(I686_CPU) case CPUCLASS_686: +#ifndef SMP printf("%d.%02d-MHz ", (i586_ctr_freq + 4999) / 1000000, ((i586_ctr_freq + 4999) / 10000) % 100); +#endif printf("686"); break; #endif diff --git a/sys/amd64/amd64/locore.S b/sys/amd64/amd64/locore.S index 63c11123f222..05ce518c0b7b 100644 --- a/sys/amd64/amd64/locore.S +++ b/sys/amd64/amd64/locore.S @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)locore.s 7.3 (Berkeley) 5/13/91 - * $Id: locore.s,v 1.84 1997/04/07 08:38:19 peter Exp $ + * $Id: locore.s,v 1.85 1997/04/22 06:55:24 jdp Exp $ * * originally from: locore.s, by William F. Jolitz * @@ -47,6 +47,9 @@ #include "opt_cpu.h" #include "opt_ddb.h" #include "opt_userconfig.h" +#include "opt_smp.h" +#include "opt_smp_privpages.h" +#include "opt_serial.h" #include <sys/errno.h> #include <sys/syscall.h> @@ -58,6 +61,10 @@ #include <machine/pmap.h> #include <machine/specialreg.h> +#if defined(SMP) && defined(SMP_PRIVPAGES) +#include <machine/apic.h> +#endif /* SMP && PRIVPAGES */ + #include "assym.s" /* @@ -111,6 +118,15 @@ _bootinfo: .space BOOTINFO_SIZE /* bootinfo that we can handle */ _KERNend: .long 0 /* phys addr end of kernel (just after bss) */ physfree: .long 0 /* phys addr of next free page */ +#if defined(SMP) && defined(SMP_PRIVPAGES) +cpu0pp: .long 0 /* phys addr cpu0 private pg */ +cpu0pt: .long 0 /* phys addr cpu0 private pt */ + + .globl _cpu0prvpage,_cpu0prvpt +_cpu0prvpage: .long 0 /* relocated version */ +_cpu0prvpt: .long 0 /* relocated version */ +#endif /* SMP && PRIVPAGES */ + .globl _IdlePTD _IdlePTD: .long 0 /* phys addr of kernel PTD */ @@ -752,6 +768,20 @@ over_symalloc: addl $KERNBASE, %esi movl %esi, R(_proc0paddr) +#if defined(SMP) && defined(SMP_PRIVPAGES) +/* Allocate cpu0's private data page */ + ALLOCPAGES(1) + movl %esi,R(cpu0pp) + addl $KERNBASE, %esi + movl %esi, R(_cpu0prvpage) /* relocated to KVM space */ + +/* Allocate cpu0's private page table for mapping priv page, apic, etc */ + ALLOCPAGES(1) + movl %esi,R(cpu0pt) + addl $KERNBASE, %esi + movl %esi, R(_cpu0prvpt) /* relocated to KVM space */ +#endif /* SMP && SMP_PRIVPAGES */ + /* Map read-only from zero to the end of the kernel text section */ xorl %eax, %eax #ifdef BDE_DEBUGGER @@ -799,6 +829,42 @@ map_read_write: movl $ISA_HOLE_LENGTH>>PAGE_SHIFT, %ecx fillkptphys($PG_RW) +#if defined(SMP) && defined(SMP_PRIVPAGES) +/* Map cpu0's private page into global KVM */ + movl R(cpu0pp), %eax + movl $1, %ecx + fillkptphys($PG_RW) + +/* Map cpu0's private page table into global KVM */ + movl R(cpu0pt), %eax + movl $1, %ecx + fillkptphys($PG_RW) + +/* Map the private page into the private page table (4K @ 0xff80000) */ + movl R(cpu0pp), %eax + movl $0, %ebx /* pte offset = 0 */ + movl $1, %ecx /* one private page coming right up */ + fillkpt(R(cpu0pt), $PG_RW) + +/* Map the default Local APIC address (4K @ 0xff801000) */ + movl $DEFAULT_APIC_BASE, %eax /* XXX just testing.. */ + movl $1, %ebx /* pte offset = 1 */ + movl $1, %ecx /* Bing! Local APIC appears */ + fillkpt(R(cpu0pt), $PG_RW|PG_N) + +/* Map the default IO APIC address (4K @ 0xff802000) */ + movl $DEFAULT_IO_APIC_BASE, %eax /* XXX just testing.. */ + movl $2, %ebx /* pte offset = 2 */ + movl $1, %ecx /* Bing! Local APIC appears */ + fillkpt(R(cpu0pt), $PG_RW|PG_N) + +/* ... and put the page table in the pde. */ + movl R(cpu0pt), %eax + movl $MPPTDI, %ebx + movl $1, %ecx + fillkpt(R(_IdlePTD), $PG_RW) +#endif /* SMP && SMP_PRIVPAGES */ + /* install a pde for temporary double map of bottom of VA */ movl R(_KPTphys), %eax xorl %ebx, %ebx diff --git a/sys/amd64/amd64/locore.s b/sys/amd64/amd64/locore.s index 63c11123f222..05ce518c0b7b 100644 --- a/sys/amd64/amd64/locore.s +++ b/sys/amd64/amd64/locore.s @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)locore.s 7.3 (Berkeley) 5/13/91 - * $Id: locore.s,v 1.84 1997/04/07 08:38:19 peter Exp $ + * $Id: locore.s,v 1.85 1997/04/22 06:55:24 jdp Exp $ * * originally from: locore.s, by William F. Jolitz * @@ -47,6 +47,9 @@ #include "opt_cpu.h" #include "opt_ddb.h" #include "opt_userconfig.h" +#include "opt_smp.h" +#include "opt_smp_privpages.h" +#include "opt_serial.h" #include <sys/errno.h> #include <sys/syscall.h> @@ -58,6 +61,10 @@ #include <machine/pmap.h> #include <machine/specialreg.h> +#if defined(SMP) && defined(SMP_PRIVPAGES) +#include <machine/apic.h> +#endif /* SMP && PRIVPAGES */ + #include "assym.s" /* @@ -111,6 +118,15 @@ _bootinfo: .space BOOTINFO_SIZE /* bootinfo that we can handle */ _KERNend: .long 0 /* phys addr end of kernel (just after bss) */ physfree: .long 0 /* phys addr of next free page */ +#if defined(SMP) && defined(SMP_PRIVPAGES) +cpu0pp: .long 0 /* phys addr cpu0 private pg */ +cpu0pt: .long 0 /* phys addr cpu0 private pt */ + + .globl _cpu0prvpage,_cpu0prvpt +_cpu0prvpage: .long 0 /* relocated version */ +_cpu0prvpt: .long 0 /* relocated version */ +#endif /* SMP && PRIVPAGES */ + .globl _IdlePTD _IdlePTD: .long 0 /* phys addr of kernel PTD */ @@ -752,6 +768,20 @@ over_symalloc: addl $KERNBASE, %esi movl %esi, R(_proc0paddr) +#if defined(SMP) && defined(SMP_PRIVPAGES) +/* Allocate cpu0's private data page */ + ALLOCPAGES(1) + movl %esi,R(cpu0pp) + addl $KERNBASE, %esi + movl %esi, R(_cpu0prvpage) /* relocated to KVM space */ + +/* Allocate cpu0's private page table for mapping priv page, apic, etc */ + ALLOCPAGES(1) + movl %esi,R(cpu0pt) + addl $KERNBASE, %esi + movl %esi, R(_cpu0prvpt) /* relocated to KVM space */ +#endif /* SMP && SMP_PRIVPAGES */ + /* Map read-only from zero to the end of the kernel text section */ xorl %eax, %eax #ifdef BDE_DEBUGGER @@ -799,6 +829,42 @@ map_read_write: movl $ISA_HOLE_LENGTH>>PAGE_SHIFT, %ecx fillkptphys($PG_RW) +#if defined(SMP) && defined(SMP_PRIVPAGES) +/* Map cpu0's private page into global KVM */ + movl R(cpu0pp), %eax + movl $1, %ecx + fillkptphys($PG_RW) + +/* Map cpu0's private page table into global KVM */ + movl R(cpu0pt), %eax + movl $1, %ecx + fillkptphys($PG_RW) + +/* Map the private page into the private page table (4K @ 0xff80000) */ + movl R(cpu0pp), %eax + movl $0, %ebx /* pte offset = 0 */ + movl $1, %ecx /* one private page coming right up */ + fillkpt(R(cpu0pt), $PG_RW) + +/* Map the default Local APIC address (4K @ 0xff801000) */ + movl $DEFAULT_APIC_BASE, %eax /* XXX just testing.. */ + movl $1, %ebx /* pte offset = 1 */ + movl $1, %ecx /* Bing! Local APIC appears */ + fillkpt(R(cpu0pt), $PG_RW|PG_N) + +/* Map the default IO APIC address (4K @ 0xff802000) */ + movl $DEFAULT_IO_APIC_BASE, %eax /* XXX just testing.. */ + movl $2, %ebx /* pte offset = 2 */ + movl $1, %ecx /* Bing! Local APIC appears */ + fillkpt(R(cpu0pt), $PG_RW|PG_N) + +/* ... and put the page table in the pde. */ + movl R(cpu0pt), %eax + movl $MPPTDI, %ebx + movl $1, %ecx + fillkpt(R(_IdlePTD), $PG_RW) +#endif /* SMP && SMP_PRIVPAGES */ + /* install a pde for temporary double map of bottom of VA */ movl R(_KPTphys), %eax xorl %ebx, %ebx diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index 9ed0b1d42d0f..d3b73299c37f 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91 - * $Id: machdep.c,v 1.237 1997/04/13 04:07:24 dyson Exp $ + * $Id: machdep.c,v 1.238 1997/04/22 06:55:26 jdp Exp $ */ #include "npx.h" @@ -44,6 +44,7 @@ #include "opt_bounce.h" #include "opt_machdep.h" #include "opt_perfmon.h" +#include "opt_smp.h" #include "opt_userconfig.h" #include <sys/param.h> @@ -107,6 +108,9 @@ #include <machine/cons.h> #include <machine/bootinfo.h> #include <machine/md_var.h> +#ifdef SMP +#include <machine/smp.h> +#endif #ifdef PERFMON #include <machine/perfmon.h> #endif @@ -209,6 +213,9 @@ cpu_startup(dummy) * Good {morning,afternoon,evening,night}. */ printf(version); +#ifdef SMP + mp_announce(); +#endif earlysetcpuclass(); startrtclock(); printcpuinfo(); @@ -344,6 +351,23 @@ again: u_map = kmem_suballoc(kernel_map, &minaddr, &maxaddr, (maxproc*UPAGES*PAGE_SIZE), FALSE); +#if defined(SMP) && defined(SMP_PRIVPAGES) + /* Per-cpu pages.. (the story so far is... subject to change) + * ========= For the per-cpu data page ======== + * 1 private data page + * 1 PDE (per-cpu PTD entry page) + * 1 PT (per-cpu page table page) + * ============ For the idle loop ============= + * 2 UPAGEs (per-cpu idle procs) + * 1 PTD (for per-cpu equiv of IdlePTD) + * ============================================ + * = total of 6 pages per cpu. The BSP reuses the ones allocated + * by locore.s during boot to remove special cases at runtime. + */ + ppage_map = kmem_suballoc(kernel_map, &minaddr, &maxaddr, + (NCPU*6*PAGE_SIZE), FALSE); +#endif + /* * Finally, allocate mbuf pool. Since mclrefcnt is an off-size * we use the more space efficient malloc in place of kmem_alloc. @@ -748,10 +772,24 @@ SYSCTL_INT(_machdep, CPU_WALLCLOCK, wall_cmos_clock, int currentldt; int _default_ldt; +#ifdef SMP +union descriptor gdt[NGDT + NCPU]; /* global descriptor table */ +#else union descriptor gdt[NGDT]; /* global descriptor table */ +#endif struct gate_descriptor idt[NIDT]; /* interrupt descriptor table */ union descriptor ldt[NLDT]; /* local descriptor table */ +#ifdef SMP +/* table descriptors - used to load tables by microp */ +struct region_descriptor r_gdt, r_idt; +#endif + +#ifdef SMP +struct i386tss SMPcommon_tss[NCPU]; /* One tss per cpu */ +struct i386tss *SMPcommon_tss_ptr[NCPU]; /* for the benefit of asmp code */ +#else struct i386tss common_tss; +#endif static struct i386tss dblfault_tss; static char dblfault_stack[PAGE_SIZE]; @@ -764,7 +802,11 @@ int gsel_tss; #endif /* software prototypes -- in more palatable form */ -struct soft_segment_descriptor gdt_segs[] = { +struct soft_segment_descriptor gdt_segs[ +#ifdef SMP + NGDT + NCPU +#endif + ] = { /* GNULL_SEL 0 Null Descriptor */ { 0x0, /* segment base address */ 0x0, /* length */ @@ -820,7 +862,12 @@ struct soft_segment_descriptor gdt_segs[] = { 0, /* unused - default 32 vs 16 bit size */ 0 /* limit granularity (byte/page units)*/ }, /* GPROC0_SEL 6 Proc 0 Tss Descriptor */ -{ (int) &common_tss, /* segment base address */ +{ +#ifdef SMP + (int) &SMPcommon_tss[0],/* segment base address */ +#else + (int) &common_tss, /* segment base address */ +#endif sizeof(struct i386tss)-1,/* length - all address space */ SDT_SYS386TSS, /* segment type */ 0, /* segment descriptor priority level */ @@ -968,8 +1015,10 @@ init386(first) int gsel_tss; #endif struct isa_device *idp; +#ifndef SMP /* table descriptors - used to load tables by microp */ struct region_descriptor r_gdt, r_idt; +#endif int pagesinbase, pagesinext; int target_page, pa_indx; int off; @@ -1002,6 +1051,18 @@ init386(first) for (x = 0; x < NGDT; x++) ssdtosd(&gdt_segs[x], &gdt[x].sd); +#ifdef SMP + /* + * Oh puke! + */ + for (x = 0; x < NCPU; x++) { + SMPcommon_tss_ptr[x] = &SMPcommon_tss[x]; + gdt_segs[NGDT + x] = gdt_segs[GPROC0_SEL]; + gdt_segs[NGDT + x].ssd_base = (int) SMPcommon_tss_ptr[x]; + ssdtosd(&gdt_segs[NGDT + x], &gdt[NGDT + x].sd); + } +#endif + /* make ldt memory segments */ /* * The data segment limit must not cover the user area because we @@ -1152,7 +1213,13 @@ init386(first) bootinfo.bi_extmem, biosextmem); } +#ifdef SMP + /* make hole for AP bootstrap code */ + pagesinbase = mp_bootaddress(biosbasemem) / PAGE_SIZE; +#else pagesinbase = biosbasemem * 1024 / PAGE_SIZE; +#endif + pagesinext = biosextmem * 1024 / PAGE_SIZE; /* @@ -1189,6 +1256,11 @@ init386(first) /* call pmap initialization to make new kernel address space */ pmap_bootstrap (first, 0); +#ifdef SMP + /* fire up the APs and APICs */ + mp_start(); +#endif + /* * Size up each available chunk of physical memory. */ @@ -1315,12 +1387,23 @@ init386(first) avail_end + off, VM_PROT_ALL, TRUE); msgbufmapped = 1; +#ifdef SMP + for(x = 0; x < NCPU; x++) { + /* make an initial tss so cpu can get interrupt stack on syscall! */ + SMPcommon_tss[x].tss_esp0 = (int) proc0.p_addr + UPAGES*PAGE_SIZE; + SMPcommon_tss[x].tss_ss0 = GSEL(GDATA_SEL, SEL_KPL) ; + SMPcommon_tss[x].tss_ioopt = (sizeof SMPcommon_tss[x]) << 16; + } + gsel_tss = GSEL(NGDT + cpunumber(), SEL_KPL); + ltr(gsel_tss); +#else /* make an initial tss so cpu can get interrupt stack on syscall! */ common_tss.tss_esp0 = (int) proc0.p_addr + UPAGES*PAGE_SIZE; common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL) ; common_tss.tss_ioopt = (sizeof common_tss) << 16; gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); ltr(gsel_tss); +#endif dblfault_tss.tss_esp = dblfault_tss.tss_esp0 = dblfault_tss.tss_esp1 = dblfault_tss.tss_esp2 = (int) &dblfault_stack[sizeof(dblfault_stack)]; @@ -1361,6 +1444,7 @@ init386(first) /* setup proc 0's pcb */ proc0.p_addr->u_pcb.pcb_flags = 0; proc0.p_addr->u_pcb.pcb_cr3 = IdlePTD; + proc0.p_addr->u_pcb.pcb_mpnest = 1; } /* diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c new file mode 100644 index 000000000000..07fca34efe56 --- /dev/null +++ b/sys/amd64/amd64/mp_machdep.c @@ -0,0 +1,1646 @@ +/* + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: mp_machdep.c,v 1.47 1997/04/26 08:11:49 peter Exp $ + */ + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" +#if defined(APIC_IO) +#if !defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#else /* APIC_IO */ +#if defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#endif /* APIC_IO */ +#define FIX_MP_TABLE_WORKS_NOT + +#include "opt_serial.h" + +#include <sys/param.h> /* for KERNBASE */ +#include <sys/types.h> +#include <sys/sysproto.h> +#include <sys/time.h> +#include <sys/systm.h> + +#include <vm/vm.h> /* for KERNBASE */ +#include <vm/vm_param.h> /* for KERNBASE */ +#include <vm/pmap.h> /* for KERNBASE */ +#include <machine/pmap.h> /* for KERNBASE */ + +#include <machine/smp.h> +#include <machine/apic.h> +#include <machine/mpapic.h> +#include <machine/cpufunc.h> +#include <machine/segments.h> +#include <machine/smptests.h> /** TEST_UPPERPRIO, TEST_DEFAULT_CONFIG */ + +#include <i386/i386/cons.h> /* cngetc() */ + +#if defined(IPI_INTS) +#include <i386/isa/isa_device.h> +#include "vector.h" +#endif /* IPI_INTS */ + +#if defined(SMP_INVLTLB) +#include <i386/isa/icu.h> +#endif /* SMP_INVLTLB */ + +#define WARMBOOT_TARGET 0 +#define WARMBOOT_OFF (KERNBASE + 0x0467) +#define WARMBOOT_SEG (KERNBASE + 0x0469) + +#define BIOS_BASE (0xf0000) +#define BIOS_SIZE (0x10000) +#define BIOS_COUNT (BIOS_SIZE/4) + +#define CMOS_REG (0x70) +#define CMOS_DATA (0x71) +#define BIOS_RESET (0x0f) +#define BIOS_WARM (0x0a) + +/* + * this code MUST be enabled here and in mpboot.s. + * it follows the very early stages of AP boot by placing values in CMOS ram. + * it NORMALLY will never be needed and thus the primitive method for enabling. + * +#define CHECK_POINTS + */ + +#if defined(CHECK_POINTS) +#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) +#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) + +#define CHECK_INIT(D); \ + CHECK_WRITE(0x34, (D)); \ + CHECK_WRITE(0x35, (D)); \ + CHECK_WRITE(0x36, (D)); \ + CHECK_WRITE(0x37, (D)); \ + CHECK_WRITE(0x38, (D)); \ + CHECK_WRITE(0x39, (D)); + +#define CHECK_PRINT(S); \ + printf("%s: %d, %d, %d, %d, %d, %d\n", \ + (S), \ + CHECK_READ(0x34), \ + CHECK_READ(0x35), \ + CHECK_READ(0x36), \ + CHECK_READ(0x37), \ + CHECK_READ(0x38), \ + CHECK_READ(0x39)); + +#else /* CHECK_POINTS */ + +#define CHECK_INIT(D) +#define CHECK_PRINT(S) + +#endif /* CHECK_POINTS */ + + +/** FIXME: what system files declare these??? */ +extern struct region_descriptor r_gdt, r_idt; + +/* global data */ +struct proc *SMPcurproc[NCPU]; +struct pcb *SMPcurpcb[NCPU]; +struct timeval SMPruntime[NCPU]; + +int mp_ncpus; /* # of CPUs, including BSP */ +int mp_naps; /* # of Applications processors */ +int mp_nbusses; /* # of busses */ +int mp_napics; /* # of IO APICs */ +int mpenabled; +int boot_cpu_id; /* designated BSP */ +vm_offset_t cpu_apic_address; +vm_offset_t io_apic_address[NAPIC]; + +u_int32_t cpu_apic_versions[NCPU]; +u_int32_t io_apic_versions[NAPIC]; + +/* + * APIC ID logical/physical mapping structures + */ +int cpu_num_to_apic_id[NCPU]; +int io_num_to_apic_id[NAPIC]; +int apic_id_to_logical[NAPICID]; + +/* + * look for MP compliant motherboard. + */ + +static u_int boot_address; +static u_int base_memory; + +static int picmode; /* 0: virtual wire mode, 1: PIC mode */ +static u_int mpfps; +static int search_for_sig(u_int32_t target, int count); +static int mp_probe(u_int base_top); +static void mp_enable(u_int boot_addr); + +#if defined(IPI_INTS) +static void ipi_intr0(void); +static void ipi_intr1(void); +static void ipi_intr2(void); +static void ipi_intr3(void); +static int +ipi_ihandler_attach(int irq, inthand2_t * func, + unsigned *maskptr, int unit); +#endif /* IPI_INTS */ + + +/* + * calculate usable address in base memory for AP trampoline code + */ +u_int +mp_bootaddress(u_int basemem) +{ + base_memory = basemem * 1024; /* convert to bytes */ + + boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ + if ((base_memory - boot_address) < bootMP_size) + boot_address -= 4096; /* not enough, lower by 4k */ + + return boot_address; +} + + +/* + * startup the SMP processors + */ +void +mp_start(void) +{ + /* look for MP capable motherboard */ + if (mp_probe(base_memory)) + mp_enable(boot_address); + else { + printf("MP FPS NOT FOUND, suggest use of 'mptable' program\n"); + panic("can't continue!\n"); + } + + /* finish pmap initialization - turn off V==P mapping at zero */ + pmap_bootstrap2(); +} + + +/* + * print various information about the SMP system hardware and setup + */ +void +mp_announce(void) +{ + int x; + + printf("FreeBSD/SMP: Multiprocessor motherboard\n"); + printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); + printf(", version: 0x%08x\n", cpu_apic_versions[0]); + for (x = 1; x <= mp_naps; ++x) { + printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); + printf(", version: 0x%08x\n", cpu_apic_versions[x]); + } + +#if defined(APIC_IO) + for (x = 0; x < mp_napics; ++x) { + printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); + printf(", version: 0x%08x\n", io_apic_versions[x]); + } +#else + printf(" Warning: APIC I/O disabled\n"); +#endif /* APIC_IO */ +} + + +/* + * AP cpu's call this to sync up protected mode. + */ +void +init_secondary(void) +{ + int gsel_tss, slot; + + r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; + r_gdt.rd_base = (int) gdt; + lgdt(&r_gdt); /* does magic intra-segment return */ + lidt(&r_idt); + lldt(_default_ldt); + + slot = NGDT + cpunumber(); + gsel_tss = GSEL(slot, SEL_KPL); + gdt[slot].sd.sd_type = SDT_SYS386TSS; + ltr(gsel_tss); + + load_cr0(0x8005003b); /* XXX! */ +} + + +#if defined(APIC_IO) +void +configure_local_apic(void) +{ + u_char byte; + u_int32_t temp; + + if (picmode) { + outb(0x22, 0x70); /* select IMCR */ + byte = inb(0x23); /* current contents */ + byte |= 0x01; /* mask external INTR */ + outb(0x23, byte); /* disconnect 8259s/NMI */ + } + /* mask the LVT1 */ + temp = apic_base[APIC_LVT1]; + temp |= APIC_LVT_M; + apic_base[APIC_LVT1] = temp; +} +#endif /* APIC_IO */ + + +/******************************************************************* + * local functions and data + */ + +static int +mp_probe(u_int base_top) +{ + int x; + u_long segment; + u_int32_t target; + + /* see if EBDA exists */ + if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { + /* search first 1K of EBDA */ + target = (u_int32_t) (segment << 4); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } else { + /*last 1K of base memory, effective 'top of base' is passed in*/ + target = (u_int32_t) (base_top - 0x400); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } + + /* search the BIOS */ + target = (u_int32_t) BIOS_BASE; + if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) + goto found; + + /* nothing found */ + mpfps = mpenabled = 0; + return 0; + +found: /* please forgive the 'goto'! */ + /* flag fact that we are running multiple processors */ + mpfps = x; + mpenabled = 1; + return 1; +} + + +/* + * start the SMP system + */ +static int parse_mp_table(void); +static void default_mp_table(int type); +static int start_all_aps(u_int boot_addr); + +#if defined(XFAST_IPI32) +#include <machine/md_var.h> +#include <i386/isa/isa_device.h> +extern void Xfastipi32(u_int, u_int, u_int, u_int); +#endif /* XFAST_IPI32 */ + +static void +mp_enable(u_int boot_addr) +{ + int x; +#if defined(APIC_IO) + int apic; + u_int ux; +#if defined(TEST_UPPERPRIO) + u_char select; /* the select register is 8 bits */ + u_int32_t flags; /* the window register is 32 bits */ +#endif /* TEST_UPPERPRIO */ +#endif /* APIC_IO */ + + /* examine the MP table for needed info */ + x = parse_mp_table(); + + /* create pages for (address common) cpu APIC and each IO APIC */ + pmap_bootstrap_apics(); + + /* can't process default configs till the CPU APIC is pmapped */ + if (x) + default_mp_table(x); + +#if defined(APIC_IO) + /* fill the LOGICAL io_apic_versions table */ + for (apic = 0; apic < mp_napics; ++apic) { + ux = io_apic_read(apic, IOAPIC_VER); + io_apic_versions[apic] = ux; + } + + /* + */ + for (apic = 0; apic < mp_napics; ++apic) + if (io_apic_setup(apic) < 0) + panic("IO APIC setup failure\n"); + +#if defined(IPI_INTS) + /* setup IPI INTerrupt mechanism */ + ipi_ihandler_attach( /* irq */ 24, + /* XXX */ (inthand2_t *) ipi_intr0, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 25, + /* XXX */ (inthand2_t *) ipi_intr1, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 26, + /* XXX */ (inthand2_t *) ipi_intr2, NULL, /* unit */ 0); +#if !defined(SMP_INVLTLB) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); +#else +#if defined(XFAST_IPI32) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); + setidt(ICU_OFFSET + 32, + Xfastipi32, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#else + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_invltlb, NULL, /* unit */ 0); +#endif /* XFAST_IPI32 */ +#endif + +#endif /* IPI_INTS */ + +#if defined(TEST_UPPERPRIO) + +#if 1 + printf("special IRQ10\n"); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~0xff; /** clear vector */ + flags |= 64; + io_apic_write(0, select, flags); +#else + printf("special IRQ10\n"); + cngetc(); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~IOART_DELMOD; /* FIXED mode */ + io_apic_write(0, select, flags); + io_apic_write(0, select + 1, boot_cpu_id << 24); +#endif /** 0/1 */ + +#endif /* TEST_UPPERPRIO */ + +#endif /* APIC_IO */ + + /* start each Application Processor */ + start_all_aps(boot_addr); +} + + +/* + * look for the MP spec signature + */ + +/* string defined by the Intel MP Spec as identifying the MP table */ +#define MP_SIG 0x5f504d5f /* _MP_ */ +#define NEXT(X) ((X) += 4) +static int +search_for_sig(u_int32_t target, int count) +{ + int x; + u_int32_t *addr = (u_int32_t *) (KERNBASE + target); + + for (x = 0; x < count; NEXT(x)) + if (addr[x] == MP_SIG) + /* make array index a byte index */ + return (target + (x * sizeof(u_int32_t))); + + return -1; +} + + +#define PROCENTRY_FLAG_EN 0x01 +#define PROCENTRY_FLAG_BP 0x02 +#define IOAPICENTRY_FLAG_EN 0x01 + +/* MP Floating Pointer Structure */ +typedef struct MPFPS { + char signature[4]; + void *pap; + u_char length; + u_char spec_rev; + u_char checksum; + u_char mpfb1; + u_char mpfb2; + u_char mpfb3; + u_char mpfb4; + u_char mpfb5; +} *mpfps_t; +/* MP Configuration Table Header */ +typedef struct MPCTH { + char signature[4]; + u_short base_table_length; + u_char spec_rev; + u_char checksum; + u_char oem_id[8]; + u_char product_id[12]; + void *oem_table_pointer; + u_short oem_table_size; + u_short entry_count; + void *apic_address; + u_short extended_table_length; + u_char extended_table_checksum; + u_char reserved; +} *mpcth_t; + + +typedef struct PROCENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char cpu_flags; + u_long cpu_signature; + u_long feature_flags; + u_long reserved1; + u_long reserved2; +} *proc_entry_ptr; + +typedef struct BUSENTRY { + u_char type; + u_char bus_id; + char bus_type[6]; +} *bus_entry_ptr; + +typedef struct IOAPICENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char apic_flags; + void *apic_address; +} *io_apic_entry_ptr; + +typedef struct INTENTRY { + u_char type; + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} *int_entry_ptr; +/* descriptions of MP basetable entries */ +typedef struct BASETABLE_ENTRY { + u_char type; + u_char length; + char name[16]; +} basetable_entry; + +static basetable_entry basetable_entry_types[] = +{ + {0, 20, "Processor"}, + {1, 8, "Bus"}, + {2, 8, "I/O APIC"}, + {3, 8, "I/O INT"}, + {4, 8, "Local INT"} +}; + +typedef struct BUSDATA { + u_char bus_id; + enum busTypes bus_type; +} bus_datum; + +typedef struct INTDATA { + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} io_int, local_int; + +typedef struct BUSTYPENAME { + u_char type; + char name[7]; +} bus_type_name; + +static bus_type_name bus_type_table[] = +{ + {CBUS, "CBUS"}, + {CBUSII, "CBUSII"}, + {EISA, "EISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {ISA, "ISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {PCI, "PCI"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {XPRESS, "XPRESS"}, + {UNKNOWN_BUSTYPE, "---"} +}; +/* from MP spec v1.4, table 5-1 */ +static int default_data[7][5] = +{ +/* nbus, id0, type0, id1, type1 */ + {1, 0, ISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {0, 255, 255, 255, 255},/* MCA not supported */ + {2, 0, ISA, 1, PCI}, + {2, 0, EISA, 1, PCI}, + {0, 255, 255, 255, 255} /* MCA not supported */ +}; + + +/* the bus data */ +bus_datum bus_data[NBUS]; + +/* the IO INT data, one entry per possible APIC INTerrupt */ +io_int io_apic_ints[NINTR]; + +static int nintrs; + +#if defined(FIX_MP_TABLE_WORKS) +static void fix_mp_table __P((void)); +#endif /* FIX_MP_TABLE_WORKS */ + +static void processor_entry __P((proc_entry_ptr entry, int *cpu)); +static void io_apic_entry __P((io_apic_entry_ptr entry, int *apic)); +static void bus_entry __P((bus_entry_ptr entry, int *bus)); +static void int_entry __P((int_entry_ptr entry, int *intr)); +static int lookup_bus_type __P((char *name)); + + +/* + * parse an Intel MP specification table + */ +static int +parse_mp_table(void) +{ + int x; + mpfps_t fps; + mpcth_t cth; + int totalSize; + void *position; + int count; + int type; + int apic, bus, cpu, intr; + + /* clear physical APIC ID to logical CPU/IO table */ + for (x = 0; x < NAPICID; ++x) + ID_TO_IO(x) = -1; + + /* clear logical CPU to APIC ID table */ + for (x = 0; x < NCPU; ++x) + CPU_TO_ID(x) = -1; + + /* clear logical IO to APIC ID table */ + for (x = 0; x < NAPIC; ++x) + IO_TO_ID(x) = -1; + + /* clear IO APIC address table */ + for (x = 0; x < NAPIC; ++x) + io_apic_address[x] = ~0; + + /* clear bus data table */ + for (x = 0; x < NBUS; ++x) + bus_data[x].bus_id = 0xff; + + /* clear IO APIC INT table */ + for (x = 0; x < NINTR; ++x) + io_apic_ints[x].int_type = 0xff; + nintrs = 0; + + /* count the BSP */ + mp_ncpus = 1; + + /* setup the cpu/apic mapping arrays */ + boot_cpu_id = -1; + + /* local pointer */ + fps = (mpfps_t) mpfps; + + /* record whether PIC or virtual-wire mode */ + picmode = (fps->mpfb2 & 0x80) ? 1 : 0; + + /* check for use of 'default' configuration */ +#if defined(TEST_DEFAULT_CONFIG) + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return TEST_DEFAULT_CONFIG; +#else + if (fps->mpfb1 != 0) { + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return fps->mpfb1; + } +#endif /* TEST_DEFAULT_CONFIG */ + + if ((cth = fps->pap) == 0) + panic("MP Configuration Table Header MISSING!\n"); + + cpu_apic_address = (vm_offset_t) cth->apic_address; + + totalSize = cth->base_table_length - sizeof(struct MPCTH); + position = (u_char *) cth + sizeof(struct MPCTH); + count = cth->entry_count; + + apic = 0; /* logical apic# start @ 0 */ + bus = 0; /* logical bus# start @ 0 */ + cpu = 1; /* logical cpu# start @ 0, BUT reserve 0 for */ + /* BSP */ + intr = 0; /* unknown */ + + /* walk the table, recording info of interest */ + while (count--) { + switch (type = *(u_char *) position) { + case 0: + processor_entry(position, &cpu); + break; + case 1: + bus_entry(position, &bus); + break; + case 2: + io_apic_entry(position, &apic); + break; + case 3: + int_entry(position, &intr); + break; + case 4: + /* int_entry(position); */ + break; + default: + panic("mpfps Base Table HOSED!\n"); + /* NOTREACHED */ + } + + totalSize -= basetable_entry_types[type].length; + (u_char *) position += basetable_entry_types[type].length; + } + + if (boot_cpu_id == -1) + panic("NO BSP found!\n"); + + /* record # of APs found */ + mp_naps = (cpu - 1); + + /* record # of busses found */ + mp_nbusses = bus; + + /* record # of IO APICs found */ + mp_napics = apic; + + /* record # of IO APICs found */ + nintrs = intr; + +#if defined(FIX_MP_TABLE_WORKS) + /* post scan cleanup */ + fix_mp_table(); +#endif /* FIX_MP_TABLE_WORKS */ + + /* report fact that its NOT a default configuration */ + return 0; +} + + +/* + * parse an Intel MP specification table + */ +#if defined(FIX_MP_TABLE_WORKS) +static void +fix_mp_table(void) +{ + int x; + int y; + int num_pci_bus; + bus_datum bus_record; + + /* + * Fix mis-numbering of the PCI bus and its INT entries if the BIOS + * did it wrong. The MP spec says that when more than 1 PCI bus + * exists the BIOS must begin with bus entries for the PCI bus and use + * actual PCI bus numbering. This implies that when only 1 PCI bus + * exists the BIOS can choose to ignore this ordering, and indeed many + * MP motherboards do ignore it. This causes a problem when the PCI + * sub-system makes requests of the MP sub-system based on PCI bus + * numbers. So here we look for the situation and renumber the + * busses and associated INTs in an effort to "make it right". + */ + + /* count the number of PCI busses */ + for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + ++num_pci_bus; + } + + /* check the 1 PCI bus case for sanity */ + if (num_pci_bus == 1) { + + /* if its in the first slot all is well */ + if (bus_data[0].bus_type == PCI) + return; + + /* mis-numbered, swap with whichever bus uses slot 0 */ + + /* locate the entry holding the PCI bus */ + for (x = 1; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + break; + } + + /* swap the bus entry records */ + bus_record = bus_data[0]; + bus_data[0] = bus_data[x]; + bus_data[x] = bus_record; + + /* swap each relavant INTerrupt entry */ + for (y = 0; y < nintrs; ++y) { + if (io_apic_ints[y].src_bus_id == x) + io_apic_ints[y].src_bus_id = 0; + else + if (io_apic_ints[y].src_bus_id == 0) + io_apic_ints[y].src_bus_id = x; + } + } + /* sanity check if more than 1 PCI bus */ + else + if (num_pci_bus > 1) { + for (x = 0; x < num_pci_bus; ++x) { + if (bus_data[x].bus_type != PCI) { + printf("bad PCI bus numbering\n"); + panic("\n"); + } + } + } +} +#endif /* FIX_MP_TABLE_WORKS */ + + +static void +processor_entry(proc_entry_ptr entry, int *cpu) +{ + int x = *cpu; + + /* check for usability */ + if (!(entry->cpu_flags & PROCENTRY_FLAG_EN)) + return; + + /* check for BSP flag */ + if (entry->cpu_flags & PROCENTRY_FLAG_BP) { + /* always give boot CPU the logical value of 0 */ + x = 0; + boot_cpu_id = entry->apic_id; + } else { + /* add another AP to list, if less than max number of CPUs */ + if (x == NCPU) { + printf("Warning: only using %d of the available CPUs!\n", x); + return; + } + ++(*cpu); + } + + CPU_TO_ID(x) = entry->apic_id; + ID_TO_CPU(entry->apic_id) = x; +} + + +static void +bus_entry(bus_entry_ptr entry, int *bus) +{ + int x, y; + char name[8]; + char c; + + if ((x = (*bus)++) == NBUS) + panic("too many busses, increase 'NBUS'\n"); + + /* encode the name into an index */ + for (y = 0; y < 6; ++y) { + if ((c = entry->bus_type[y]) == ' ') + break; + name[y] = c; + } + name[y] = '\0'; + + if ((y = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) + panic("unknown bus type: '%s'\n", name); + + bus_data[x].bus_id = entry->bus_id; + bus_data[x].bus_type = y; +} + + +static void +io_apic_entry(io_apic_entry_ptr entry, int *apic) +{ + int x; + + if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) + return; + + if ((x = (*apic)++) == NAPIC) + panic("too many APICs, increase 'NAPIC'\n"); + + IO_TO_ID(x) = entry->apic_id; + ID_TO_IO(entry->apic_id) = x; + + io_apic_address[x] = (vm_offset_t) entry->apic_address; +} + + +static int +lookup_bus_type(char *name) +{ + int x; + + for (x = 0; x < MAX_BUSTYPE; ++x) + if (strcmp(bus_type_table[x].name, name) == 0) + return bus_type_table[x].type; + + return UNKNOWN_BUSTYPE; +} + + +static void +int_entry(int_entry_ptr entry, int *intr) +{ + int x; + + if ((x = (*intr)++) == NINTR) + panic("too many INTs, increase 'NINTR'\n"); + + io_apic_ints[x].int_type = entry->int_type; + io_apic_ints[x].int_flags = entry->int_flags; + io_apic_ints[x].src_bus_id = entry->src_bus_id; + io_apic_ints[x].src_bus_irq = entry->src_bus_irq; + io_apic_ints[x].dst_apic_id = entry->dst_apic_id; + io_apic_ints[x].dst_apic_int = entry->dst_apic_int; +} + + +static int +apic_int_is_bus_type(int intr, int bus_type) +{ + int bus; + + for (bus = 0; bus < mp_nbusses; ++bus) + if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) + && ((int) bus_data[bus].bus_type == bus_type)) + return 1; + + return 0; +} + + +/* + * determine which APIC pin an ISA INT is attached to. + */ +#define INTTYPE(I) (io_apic_ints[(I)].int_type) +#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) + +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_isa_apic_irq(int isaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (isaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == isaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, ISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin an EISA INT is attached to. + */ +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_eisa_apic_irq(int eisaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (eisaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == eisaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, EISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin a PCI INT is attached to. + */ +#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) +#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) +#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) +int +get_pci_apic_irq(int pciBus, int pciDevice, int pciInt) +{ + int intr; + + --pciInt; /* zero based */ + + for (intr = 0; intr < nintrs; ++intr) /* search each record */ + if ((INTTYPE(intr) == 0) +#if defined(FIX_MP_TABLE_WORKS) + && (SRCBUSID(intr) == pciBus) +#endif /* FIX_MP_TABLE_WORKS */ + && (SRCBUSDEVICE(intr) == pciDevice) + && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, PCI)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSLINE +#undef SRCBUSDEVICE +#undef SRCBUSID + +#undef INTPIN +#undef INTTYPE + + +int +undirect_pci_irq(int rirq) +{ +#if defined(READY) + printf("Freeing irq %d for ISA cards.\n", rirq); + /** FIXME: tickle the MB redirector chip */ + return ???; +#else + printf("Freeing (NOT implimented) irq %d for ISA cards.\n", rirq); + return 0; +#endif /* READY */ +} + + +/* + * given a bus ID, return: + * the bus type if found + * -1 if NOT found + */ +int +apic_bus_type(int id) +{ + int x; + + for (x = 0; x < mp_nbusses; ++x) + if (bus_data[x].bus_id == id) + return bus_data[x].bus_type; + + return -1; +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus ID if found + * -1 if NOT found + */ +int +apic_src_bus_id(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_id); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus IRQ if found + * -1 if NOT found + */ +int +apic_src_bus_irq(int apic, int pin) +{ + int x; + + for (x = 0; x < nintrs; x++) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_irq); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated INTerrupt type if found + * -1 if NOT found + */ +int +apic_int_type(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_type); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated trigger mode if found + * -1 if NOT found + */ +int +apic_trigger(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return ((io_apic_ints[x].int_flags >> 2) & 0x03); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated 'active' level if found + * -1 if NOT found + */ +int +apic_polarity(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_flags & 0x03); + + return -1; /* NOT found */ +} + + +/* + * set data according to MP defaults + * FIXME: probably not complete yet... + */ +static void +default_mp_table(int type) +{ + int ap_cpu_id; +#if defined(APIC_IO) + u_int32_t ux; + int io_apic_id; + int pin; +#endif /* APIC_IO */ + +#if 0 + printf(" MP default config type: %d\n", type); + switch (type) { + case 1: + printf(" bus: ISA, APIC: 82489DX\n"); + break; + case 2: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 3: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 4: + printf(" bus: MCA, APIC: 82489DX\n"); + break; + case 5: + printf(" bus: ISA+PCI, APIC: Integrated\n"); + break; + case 6: + printf(" bus: EISA+PCI, APIC: Integrated\n"); + break; + case 7: + printf(" bus: MCA+PCI, APIC: Integrated\n"); + break; + default: + printf(" future type\n"); + break; + /* NOTREACHED */ + } +#endif /* 0 */ + + boot_cpu_id = (apic_base[APIC_ID] & APIC_ID_MASK) >> 24; + ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; + + /* BSP */ + CPU_TO_ID(0) = boot_cpu_id; + ID_TO_CPU(boot_cpu_id) = 0; + + /* one and only AP */ + CPU_TO_ID(1) = ap_cpu_id; + ID_TO_CPU(ap_cpu_id) = 1; + mp_naps = 1; + + /* one and only IO APIC */ +#if defined(APIC_IO) + io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; + + /* + * sanity check, refer to MP spec section 3.6.6, last paragraph + * necessary as some hardware isn't properly setting up the IO APIC + */ +#if defined(REALLY_ANAL_IOAPICID_VALUE) + if (io_apic_id != 2) { +#else + if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { +#endif /* REALLY_ANAL_IOAPICID_VALUE */ + ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ + ux &= ~APIC_ID_MASK; /* clear the ID field */ + ux |= 0x02000000; /* set it to '2' */ + io_apic_write(0, IOAPIC_ID, ux); /* write new value */ + ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ + if ((ux & APIC_ID_MASK) != 0x02000000) + panic("Problem: can't control IO APIC ID, reg: 0x%08x\n", ux); + io_apic_id = 2; + } + IO_TO_ID(0) = io_apic_id; + ID_TO_IO(io_apic_id) = 0; + mp_napics = 1; +#else + mp_napics = 0; +#endif /* APIC_IO */ + + /* fill out bus entries */ + switch (type) { + case 1: + case 2: + case 3: + case 5: + case 6: + mp_nbusses = default_data[type - 1][0]; + bus_data[0].bus_id = default_data[type - 1][1]; + bus_data[0].bus_type = default_data[type - 1][2]; + bus_data[1].bus_id = default_data[type - 1][3]; + bus_data[1].bus_type = default_data[type - 1][4]; + break; + + /* case 4: case 7: MCA NOT supported */ + default: /* illegal/reserved */ + panic("BAD default MP config: %d\n", type); + } + +#if defined(APIC_IO) + /* general cases from MP v1.4, table 5-2 */ + for (pin = 0; pin < 16; ++pin) { + io_apic_ints[pin].int_type = 0; + io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ + io_apic_ints[pin].src_bus_id = 0; + io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ + io_apic_ints[pin].dst_apic_id = io_apic_id; + io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ + } + + /* special cases from MP v1.4, table 5-2 */ + if (type == 2) { + io_apic_ints[2].int_type = 0xff; /* N/C */ + io_apic_ints[13].int_type = 0xff; /* N/C */ +#if !defined(APIC_MIXED_MODE) + /** FIXME: ??? */ + panic("sorry, can't support type 2 default yet\n"); +#endif /* APIC_MIXED_MODE */ + } else + io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ + + if (type == 7) + io_apic_ints[0].int_type = 0xff; /* N/C */ + else + io_apic_ints[0].int_type = 3; /* vectored 8259 */ + + nintrs = 16; +#endif /* APIC_IO */ +} + + +static void install_ap_tramp(u_int boot_addr); +static int start_ap(int logicalCpu, u_int boot_addr); + +/* + * start each AP in our list + */ +static int +start_all_aps(u_int boot_addr) +{ + int x; + u_char mpbiosreason; + u_long mpbioswarmvec; + + /** + * NOTE: this needs further thought: + * where does it get released? + * should it be set to empy? + * + * get the initial mp_lock with a count of 1 for the BSP + */ + mp_lock = (apic_base[APIC_ID] & APIC_ID_MASK) + 1; + + /* initialize BSP's local APIC */ + apic_initialize(1); + + /* install the AP 1st level boot code */ + install_ap_tramp(boot_addr); + + /* save the current value of the warm-start vector */ + mpbioswarmvec = *((u_long *) WARMBOOT_OFF); + outb(CMOS_REG, BIOS_RESET); + mpbiosreason = inb(CMOS_DATA); + + /* start each AP */ + for (x = 1; x <= mp_naps; ++x) { + + /* setup a vector to our boot code */ + *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; + *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ + + /* attempt to start the Application Processor */ + CHECK_INIT(99); /* setup checkpoints */ + if (!start_ap(x, boot_addr)) { + printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); + CHECK_PRINT("trace"); /* show checkpoints */ + /* + * better panic as the AP may be running loose + * somewhere + */ + printf("panic y/n? [n] "); + if (cngetc() != 'n') + panic("bye-bye\n"); + } + CHECK_PRINT("trace"); /* show checkpoints */ + + /* record its version info */ + cpu_apic_versions[x] = cpu_apic_versions[0]; + } + + /* fill in our (BSP) APIC version */ + cpu_apic_versions[0] = apic_base[APIC_VER]; + + /* restore the warmstart vector */ + *(u_long *) WARMBOOT_OFF = mpbioswarmvec; + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, mpbiosreason); + + /* number of APs actually started */ + return mp_ncpus - 1; +} + + +/* + * load the 1st level AP boot code into base memory. + */ + +/* targets for relocation */ +extern void bigJump(void); +extern void bootCodeSeg(void); +extern void bootDataSeg(void); +extern void MPentry(void); +extern u_int MP_GDT; +extern u_int mp_gdtbase; + +static void +install_ap_tramp(u_int boot_addr) +{ + int x; + int size = *(int *) ((u_long) & bootMP_size); + u_char *src = (u_char *) ((u_long) bootMP); + u_char *dst = (u_char *) boot_addr + KERNBASE; + u_int boot_base = (u_int) bootMP; + u_int8_t *dst8; + u_int16_t *dst16; + u_int32_t *dst32; + + for (x = 0; x < size; ++x) + *dst++ = *src++; + + /* + * modify addresses in code we just moved to basemem. unfortunately we + * need fairly detailed info about mpboot.s for this to work. changes + * to mpboot.s might require changes here. + */ + + /* boot code is located in KERNEL space */ + dst = (u_char *) boot_addr + KERNBASE; + + /* modify the lgdt arg */ + dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); + *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); + + /* modify the ljmp target for MPentry() */ + dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); + *dst32 = ((u_int) MPentry - KERNBASE); + + /* modify the target for boot code segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; + + /* modify the target for boot data segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; +} + + +/* + * this function starts the AP (application processor) identified + * by the APIC ID 'physicalCpu'. It does quite a "song and dance" + * to accomplish this. This is necessary because of the nuances + * of the different hardware we might encounter. It ain't pretty, + * but it seems to work. + */ +static int +start_ap(int logical_cpu, u_int boot_addr) +{ + int physical_cpu; + int vector; + int cpus; + u_long icr_lo, icr_hi; + + /* get the PHYSICAL APIC ID# */ + physical_cpu = CPU_TO_ID(logical_cpu); + + /* calculate the vector */ + vector = (boot_addr >> 12) & 0xff; + + /* used as a watchpoint to signal AP startup */ + cpus = mp_ncpus; + + /* + * first we do an INIT/RESET IPI this INIT IPI might be run, reseting + * and running the target CPU. OR this INIT IPI might be latched (P5 + * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be + * ignored. + */ + + /* setup the address for the target AP */ + icr_hi = apic_base[APIC_ICR_HI] & ~APIC_ID_MASK; + icr_hi |= (physical_cpu << 24); + apic_base[APIC_ICR_HI] = icr_hi; + + /* do an INIT IPI: assert RESET */ + icr_lo = apic_base[APIC_ICR_LOW] & 0xfff00000; + apic_base[APIC_ICR_LOW] = icr_lo | 0x0000c500; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* do an INIT IPI: deassert RESET */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00008500; + + /* wait for pending status end */ + u_sleep(10000); /* wait ~10mS */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* + * next we do a STARTUP IPI: the previous INIT IPI might still be + * latched, (P5 bug) this 1st STARTUP would then terminate + * immediately, and the previously started INIT IPI would continue. OR + * the previous INIT IPI has already run. and this STARTUP IPI will + * run. OR the previous INIT IPI was ignored. and this STARTUP IPI + * will run. + */ + + /* do a STARTUP IPI */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* + * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF + * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR + * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is + * recognized after hardware RESET or INIT IPI. + */ + + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* wait for it to start */ + set_apic_timer(5000000);/* == 5 seconds */ + while (read_apic_timer()) + if (mp_ncpus > cpus) + return 1; /* return SUCCESS */ + + return 0; /* return FAILURE */ +} + + +#if defined(IPI_INTS) + +static void +ipi_intr0(void) +{ + printf("IPI 0\n"); +} + + +static void +ipi_intr1(void) +{ + printf("IPI 1\n"); +} + + +static void +ipi_intr2(void) +{ + printf("IPI 2\n"); +} + + +static void +ipi_intr3(void) +{ + printf("IPI 3\n"); +} + +/*----------------------------------------------------------------------- +** +** Register an interupt handler for an IPI. +** (Stolen from the PCI<->ISA glue code) +** +**----------------------------------------------------------------------- +*/ + +static int +ipi_ihandler_attach(int irq, inthand2_t * func, unsigned *maskptr, int unit) +{ + char buf[16]; + char *cp; + int free_id, id, result; + + sprintf(buf, "ipi irq%d", irq); + for (cp = intrnames, free_id = 0, id = 0; id < NR_DEVICES; id++) { + if (strcmp(cp, buf) == 0) + break; + if (free_id <= 0 && strcmp(cp, "ipi irqnn") == 0) + free_id = id; + while (*cp++ != '\0'); + } + if (id == NR_DEVICES) { + id = free_id; + if (id == 0) { + /* + * All ipi irq counters are in use, perhaps because + * config is old so there aren't any. Abuse the clk0 + * counter. + */ + printf( + "ipi_ihandler_attach: counting ipi irq%d's as clk0 irqs\n", + irq); + } + } + result = register_intr( + irq, /* isa irq */ + id, /* device id */ + 0, /* flags? */ + func, /* handler */ + maskptr, /* mask pointer */ + unit); /* handler arg */ + + if (result) { + printf("WARNING: ipi_ihandler_attach: result=%d\n", result); + return (result); + }; + + return (0); +} +#endif /* IPI_INTS */ + + +#ifdef SMP_INVLTLB +/* + * Flush the TLB on all other CPU's + * + * XXX: Needs to handshake and wait for completion before proceding. + */ + +void +smp_invltlb() +{ + if (smp_active) { + if (invldebug & 2) +#if defined(XFAST_IPI32) + all_but_self_ipi(ICU_OFFSET + 32); +#else + all_but_self_ipi(ICU_OFFSET + 27); +#endif /* XFAST_IPI32 */ + } +} + +void +invlpg(u_int addr) +{ + __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); + smp_invltlb(); +} + +void +invltlb(void) +{ + u_long temp; + /* + * This should be implemented as load_cr3(rcr3()) when load_cr3() is + * inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); + smp_invltlb(); +} +/* + * Handles recieving an "IRQ 27", the invalidate tlb IPI.. + */ +void +ipi_invltlb(void) +{ + u_long temp; + + if (invldebug & 4) { + /* + * This should be implemented as load_cr3(rcr3()) when + * load_cr3() is inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) + :: "memory"); + + } +} +#endif /* SMP_INVLTLB */ diff --git a/sys/amd64/amd64/mpboot.S b/sys/amd64/amd64/mpboot.S new file mode 100644 index 000000000000..e43b25043b70 --- /dev/null +++ b/sys/amd64/amd64/mpboot.S @@ -0,0 +1,317 @@ +/* + * Copyright (c) 1995, Jack F. Vogel + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by Jack F. Vogel + * 4. The name of the developer may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * mpboot.s: FreeBSD machine support for the Intel MP Spec + * multiprocessor systems. + * + * $Id: mpboot.s,v 1.18 1997/04/25 03:11:27 fsmp Exp $ + */ + + +#include <machine/asmacros.h> /* miscellaneous asm macros */ +#include <machine/apic.h> +#include <machine/specialreg.h> + +#include "assym.s" + +/* + * this code MUST be enabled here and in mp_machdep.c + * it follows the very early stages of AP boot by placing values in CMOS ram. + * it NORMALLY will never be needed and thus the primitive method for enabling. + * +#define CHECK_POINTS + */ + +#if defined(CHECK_POINTS) + +#define CMOS_REG (0x70) +#define CMOS_DATA (0x71) + +#define CHECKPOINT(A,D) \ + movb $(A),%al ; \ + outb %al,$CMOS_REG ; \ + movb $(D),%al ; \ + outb %al,$CMOS_DATA + +#else + +#define CHECKPOINT(A,D) + +#endif /* CHECK_POINTS */ + + +/* + * the APs enter here from their trampoline code (bootMP, below) + */ + .align 4 + +NON_GPROF_ENTRY(MPentry) + CHECKPOINT(0x36, 3) + movl $mp_stk-KERNBASE,%esp /* mp boot stack end loc. */ + /* Now enable paging mode */ + movl _IdlePTD-KERNBASE, %eax + movl %eax,%cr3 + movl %cr0,%eax + orl $CR0_PE|CR0_PG,%eax /* enable paging */ + movl $0x80000011,%eax + movl %eax,%cr0 /* let the games begin! */ + movl $mp_stk,%esp /* mp boot stack end loc. */ + + pushl $mp_begin /* jump to high mem */ + ret + + /* + * Wait for the booting CPU to signal startup + */ +mp_begin: /* now running relocated at KERNBASE */ + CHECKPOINT(0x37, 4) + call _init_secondary /* load i386 tables */ + CHECKPOINT(0x38, 5) + + /* disable the APIC, just to be SURE */ + movl _apic_base, %esi + movl APIC_SVR(%esi), %eax /* get spurious vector reg. */ + andl $~APIC_SVR_SWEN, %eax /* clear software enable bit */ + movl %eax, APIC_SVR(%esi) + + /* signal our startup to the BSP */ + movl APIC_VER(%esi), %eax /* our version reg contents */ + movl %eax, _cpu_apic_versions /* into [ 0 ] */ + incl _mp_ncpus /* signal BSP */ + + /* One at a time, we are running on the shared mp_stk */ + /* This is the Intel reccomended semaphore method */ +#define BL_SET 0xff +#define BL_CLR 0x00 + movb $BL_SET, %al +1: + xchgb %al, bootlock /* xchg is implicitly locked */ + cmpb $BL_SET, %al /* was is set? */ + jz 1b /* yes, keep trying... */ + CHECKPOINT(0x39, 6) + + /* Now, let's do some REAL WORK :-) */ + call _secondary_main +/* NOT REACHED */ +2: hlt + jmp 2b + +/* + * Let a CPU past the semaphore so it can use mp_stk + */ +ENTRY(boot_unlock) + movb $BL_CLR, %al + xchgb %al, bootlock /* xchg is implicitly locked */ + ret + +/* + * This is the embedded trampoline or bootstrap that is + * copied into 'real-mode' low memory, it is where the + * secondary processor "wakes up". When it is executed + * the processor will eventually jump into the routine + * MPentry, which resides in normal kernel text above + * 1Meg. -jackv + */ + +#define data32 .byte 0x66 +#define addr32 .byte 0x67 + + .data + ALIGN_DATA /* just to be sure */ + +BOOTMP1: + +NON_GPROF_ENTRY(bootMP) + cli + CHECKPOINT(0x34, 1) + /* First guarantee a 'clean slate' */ + data32 + xorl %eax, %eax + data32 + movl %eax, %ebx + data32 + movl %eax, %ecx + data32 + movl %eax, %edx + data32 + movl %eax, %esi + data32 + movl %eax, %edi + + /* set up data segments */ + mov %cs, %ax + mov %ax, %ds + mov %ax, %es + mov %ax, %fs + mov %ax, %gs + mov %ax, %ss + mov $(boot_stk-_bootMP), %sp + + /* Now load the global descriptor table */ + addr32 + data32 + /* XXX: sigh: lgdt MP_GDTptr-_bootMP GAS BUG! */ + .byte 0x0f, 0x01, 0x15 /* XXX hand assemble! */ + .long MP_GDTptr-_bootMP /* XXX hand assemble! */ + + /* Enable protected mode */ + data32 + movl %cr0, %eax + data32 + orl $CR0_PE, %eax + data32 + movl %eax, %cr0 + + /* + * make intrasegment jump to flush the processor pipeline and + * reload CS register + */ + data32 + pushl $0x18 + data32 + pushl $(protmode-_bootMP) + data32 + lret + +protmode: + CHECKPOINT(0x35, 2) + + /* + * we are NOW running for the first time with %eip + * having the full physical address, BUT we still + * are using a segment descriptor with the origin + * not matching the booting kernel. + * + * SO NOW... for the BIG Jump into kernel's segment + * and physical text above 1 Meg. + */ + mov $0x10, %ebx + movw %bx, %ds + movw %bx, %es + movw %bx, %fs + movw %bx, %gs + movw %bx, %ss + + .globl _bigJump +_bigJump: + /* this will be modified by mpInstallTramp() */ + ljmp $0x08, $0 /* far jmp to MPentry() */ + +dead: hlt /* We should never get here */ + jmp dead + +/* + * MP boot strap Global Descriptor Table + */ + .align 4 + .globl _MP_GDT + .globl _bootCodeSeg + .globl _bootDataSeg +_MP_GDT: + +nulldesc: /* offset = 0x0 */ + + .word 0x0 + .word 0x0 + .byte 0x0 + .byte 0x0 + .byte 0x0 + .byte 0x0 + +kernelcode: /* offset = 0x08 */ + + .word 0xffff /* segment limit 0..15 */ + .word 0x0000 /* segment base 0..15 */ + .byte 0x0 /* segment base 16..23; set for 0K */ + .byte 0x9f /* flags; Type */ + .byte 0xcf /* flags; Limit */ + .byte 0x0 /* segment base 24..32 */ + +kerneldata: /* offset = 0x10 */ + + .word 0xffff /* segment limit 0..15 */ + .word 0x0000 /* segment base 0..15 */ + .byte 0x0 /* segment base 16..23; set for 0k */ + .byte 0x93 /* flags; Type */ + .byte 0xcf /* flags; Limit */ + .byte 0x0 /* segment base 24..32 */ + +bootcode: /* offset = 0x18 */ + + .word 0xffff /* segment limit 0..15 */ +_bootCodeSeg: /* this will be modified by mpInstallTramp() */ + .word 0x0000 /* segment base 0..15 */ + .byte 0x00 /* segment base 16...23; set for 0x000xx000 */ + .byte 0x9e /* flags; Type */ + .byte 0xcf /* flags; Limit */ + .byte 0x0 /*segment base 24..32 */ + +bootdata: /* offset = 0x20 */ + + .word 0xffff +_bootDataSeg: /* this will be modified by mpInstallTramp() */ + .word 0x0000 /* segment base 0..15 */ + .byte 0x00 /* segment base 16...23; set for 0x000xx000 */ + .byte 0x92 + .byte 0xcf + .byte 0x0 + +/* + * GDT pointer for the lgdt call + */ + .globl _mp_gdtbase + +MP_GDTptr: +_mp_gdtlimit: + .word 0x0028 +_mp_gdtbase: /* this will be modified by mpInstallTramp() */ + .long 0 + + .space 0x100 /* space for boot_stk - 1st temporary stack */ +boot_stk: + +BOOTMP2: + .globl _bootMP_size +_bootMP_size: + .long BOOTMP2 - BOOTMP1 + + /* + * Temporary stack used while booting AP's + * It is protected by: + * 1: only one cpu is started at a time and it ends up waiting + * for smp_active before continuing. + * 2: Once smp_active != 0; further access is limited by _bootlock. + */ + .globl mp_stk + .space 0x2000 /* space for mp_stk - 2nd temporary stack */ +mp_stk: + + .globl bootlock +bootlock: .byte BL_SET diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c new file mode 100644 index 000000000000..07fca34efe56 --- /dev/null +++ b/sys/amd64/amd64/mptable.c @@ -0,0 +1,1646 @@ +/* + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: mp_machdep.c,v 1.47 1997/04/26 08:11:49 peter Exp $ + */ + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" +#if defined(APIC_IO) +#if !defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#else /* APIC_IO */ +#if defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#endif /* APIC_IO */ +#define FIX_MP_TABLE_WORKS_NOT + +#include "opt_serial.h" + +#include <sys/param.h> /* for KERNBASE */ +#include <sys/types.h> +#include <sys/sysproto.h> +#include <sys/time.h> +#include <sys/systm.h> + +#include <vm/vm.h> /* for KERNBASE */ +#include <vm/vm_param.h> /* for KERNBASE */ +#include <vm/pmap.h> /* for KERNBASE */ +#include <machine/pmap.h> /* for KERNBASE */ + +#include <machine/smp.h> +#include <machine/apic.h> +#include <machine/mpapic.h> +#include <machine/cpufunc.h> +#include <machine/segments.h> +#include <machine/smptests.h> /** TEST_UPPERPRIO, TEST_DEFAULT_CONFIG */ + +#include <i386/i386/cons.h> /* cngetc() */ + +#if defined(IPI_INTS) +#include <i386/isa/isa_device.h> +#include "vector.h" +#endif /* IPI_INTS */ + +#if defined(SMP_INVLTLB) +#include <i386/isa/icu.h> +#endif /* SMP_INVLTLB */ + +#define WARMBOOT_TARGET 0 +#define WARMBOOT_OFF (KERNBASE + 0x0467) +#define WARMBOOT_SEG (KERNBASE + 0x0469) + +#define BIOS_BASE (0xf0000) +#define BIOS_SIZE (0x10000) +#define BIOS_COUNT (BIOS_SIZE/4) + +#define CMOS_REG (0x70) +#define CMOS_DATA (0x71) +#define BIOS_RESET (0x0f) +#define BIOS_WARM (0x0a) + +/* + * this code MUST be enabled here and in mpboot.s. + * it follows the very early stages of AP boot by placing values in CMOS ram. + * it NORMALLY will never be needed and thus the primitive method for enabling. + * +#define CHECK_POINTS + */ + +#if defined(CHECK_POINTS) +#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) +#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) + +#define CHECK_INIT(D); \ + CHECK_WRITE(0x34, (D)); \ + CHECK_WRITE(0x35, (D)); \ + CHECK_WRITE(0x36, (D)); \ + CHECK_WRITE(0x37, (D)); \ + CHECK_WRITE(0x38, (D)); \ + CHECK_WRITE(0x39, (D)); + +#define CHECK_PRINT(S); \ + printf("%s: %d, %d, %d, %d, %d, %d\n", \ + (S), \ + CHECK_READ(0x34), \ + CHECK_READ(0x35), \ + CHECK_READ(0x36), \ + CHECK_READ(0x37), \ + CHECK_READ(0x38), \ + CHECK_READ(0x39)); + +#else /* CHECK_POINTS */ + +#define CHECK_INIT(D) +#define CHECK_PRINT(S) + +#endif /* CHECK_POINTS */ + + +/** FIXME: what system files declare these??? */ +extern struct region_descriptor r_gdt, r_idt; + +/* global data */ +struct proc *SMPcurproc[NCPU]; +struct pcb *SMPcurpcb[NCPU]; +struct timeval SMPruntime[NCPU]; + +int mp_ncpus; /* # of CPUs, including BSP */ +int mp_naps; /* # of Applications processors */ +int mp_nbusses; /* # of busses */ +int mp_napics; /* # of IO APICs */ +int mpenabled; +int boot_cpu_id; /* designated BSP */ +vm_offset_t cpu_apic_address; +vm_offset_t io_apic_address[NAPIC]; + +u_int32_t cpu_apic_versions[NCPU]; +u_int32_t io_apic_versions[NAPIC]; + +/* + * APIC ID logical/physical mapping structures + */ +int cpu_num_to_apic_id[NCPU]; +int io_num_to_apic_id[NAPIC]; +int apic_id_to_logical[NAPICID]; + +/* + * look for MP compliant motherboard. + */ + +static u_int boot_address; +static u_int base_memory; + +static int picmode; /* 0: virtual wire mode, 1: PIC mode */ +static u_int mpfps; +static int search_for_sig(u_int32_t target, int count); +static int mp_probe(u_int base_top); +static void mp_enable(u_int boot_addr); + +#if defined(IPI_INTS) +static void ipi_intr0(void); +static void ipi_intr1(void); +static void ipi_intr2(void); +static void ipi_intr3(void); +static int +ipi_ihandler_attach(int irq, inthand2_t * func, + unsigned *maskptr, int unit); +#endif /* IPI_INTS */ + + +/* + * calculate usable address in base memory for AP trampoline code + */ +u_int +mp_bootaddress(u_int basemem) +{ + base_memory = basemem * 1024; /* convert to bytes */ + + boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ + if ((base_memory - boot_address) < bootMP_size) + boot_address -= 4096; /* not enough, lower by 4k */ + + return boot_address; +} + + +/* + * startup the SMP processors + */ +void +mp_start(void) +{ + /* look for MP capable motherboard */ + if (mp_probe(base_memory)) + mp_enable(boot_address); + else { + printf("MP FPS NOT FOUND, suggest use of 'mptable' program\n"); + panic("can't continue!\n"); + } + + /* finish pmap initialization - turn off V==P mapping at zero */ + pmap_bootstrap2(); +} + + +/* + * print various information about the SMP system hardware and setup + */ +void +mp_announce(void) +{ + int x; + + printf("FreeBSD/SMP: Multiprocessor motherboard\n"); + printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); + printf(", version: 0x%08x\n", cpu_apic_versions[0]); + for (x = 1; x <= mp_naps; ++x) { + printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); + printf(", version: 0x%08x\n", cpu_apic_versions[x]); + } + +#if defined(APIC_IO) + for (x = 0; x < mp_napics; ++x) { + printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); + printf(", version: 0x%08x\n", io_apic_versions[x]); + } +#else + printf(" Warning: APIC I/O disabled\n"); +#endif /* APIC_IO */ +} + + +/* + * AP cpu's call this to sync up protected mode. + */ +void +init_secondary(void) +{ + int gsel_tss, slot; + + r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; + r_gdt.rd_base = (int) gdt; + lgdt(&r_gdt); /* does magic intra-segment return */ + lidt(&r_idt); + lldt(_default_ldt); + + slot = NGDT + cpunumber(); + gsel_tss = GSEL(slot, SEL_KPL); + gdt[slot].sd.sd_type = SDT_SYS386TSS; + ltr(gsel_tss); + + load_cr0(0x8005003b); /* XXX! */ +} + + +#if defined(APIC_IO) +void +configure_local_apic(void) +{ + u_char byte; + u_int32_t temp; + + if (picmode) { + outb(0x22, 0x70); /* select IMCR */ + byte = inb(0x23); /* current contents */ + byte |= 0x01; /* mask external INTR */ + outb(0x23, byte); /* disconnect 8259s/NMI */ + } + /* mask the LVT1 */ + temp = apic_base[APIC_LVT1]; + temp |= APIC_LVT_M; + apic_base[APIC_LVT1] = temp; +} +#endif /* APIC_IO */ + + +/******************************************************************* + * local functions and data + */ + +static int +mp_probe(u_int base_top) +{ + int x; + u_long segment; + u_int32_t target; + + /* see if EBDA exists */ + if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { + /* search first 1K of EBDA */ + target = (u_int32_t) (segment << 4); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } else { + /*last 1K of base memory, effective 'top of base' is passed in*/ + target = (u_int32_t) (base_top - 0x400); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } + + /* search the BIOS */ + target = (u_int32_t) BIOS_BASE; + if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) + goto found; + + /* nothing found */ + mpfps = mpenabled = 0; + return 0; + +found: /* please forgive the 'goto'! */ + /* flag fact that we are running multiple processors */ + mpfps = x; + mpenabled = 1; + return 1; +} + + +/* + * start the SMP system + */ +static int parse_mp_table(void); +static void default_mp_table(int type); +static int start_all_aps(u_int boot_addr); + +#if defined(XFAST_IPI32) +#include <machine/md_var.h> +#include <i386/isa/isa_device.h> +extern void Xfastipi32(u_int, u_int, u_int, u_int); +#endif /* XFAST_IPI32 */ + +static void +mp_enable(u_int boot_addr) +{ + int x; +#if defined(APIC_IO) + int apic; + u_int ux; +#if defined(TEST_UPPERPRIO) + u_char select; /* the select register is 8 bits */ + u_int32_t flags; /* the window register is 32 bits */ +#endif /* TEST_UPPERPRIO */ +#endif /* APIC_IO */ + + /* examine the MP table for needed info */ + x = parse_mp_table(); + + /* create pages for (address common) cpu APIC and each IO APIC */ + pmap_bootstrap_apics(); + + /* can't process default configs till the CPU APIC is pmapped */ + if (x) + default_mp_table(x); + +#if defined(APIC_IO) + /* fill the LOGICAL io_apic_versions table */ + for (apic = 0; apic < mp_napics; ++apic) { + ux = io_apic_read(apic, IOAPIC_VER); + io_apic_versions[apic] = ux; + } + + /* + */ + for (apic = 0; apic < mp_napics; ++apic) + if (io_apic_setup(apic) < 0) + panic("IO APIC setup failure\n"); + +#if defined(IPI_INTS) + /* setup IPI INTerrupt mechanism */ + ipi_ihandler_attach( /* irq */ 24, + /* XXX */ (inthand2_t *) ipi_intr0, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 25, + /* XXX */ (inthand2_t *) ipi_intr1, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 26, + /* XXX */ (inthand2_t *) ipi_intr2, NULL, /* unit */ 0); +#if !defined(SMP_INVLTLB) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); +#else +#if defined(XFAST_IPI32) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); + setidt(ICU_OFFSET + 32, + Xfastipi32, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#else + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_invltlb, NULL, /* unit */ 0); +#endif /* XFAST_IPI32 */ +#endif + +#endif /* IPI_INTS */ + +#if defined(TEST_UPPERPRIO) + +#if 1 + printf("special IRQ10\n"); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~0xff; /** clear vector */ + flags |= 64; + io_apic_write(0, select, flags); +#else + printf("special IRQ10\n"); + cngetc(); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~IOART_DELMOD; /* FIXED mode */ + io_apic_write(0, select, flags); + io_apic_write(0, select + 1, boot_cpu_id << 24); +#endif /** 0/1 */ + +#endif /* TEST_UPPERPRIO */ + +#endif /* APIC_IO */ + + /* start each Application Processor */ + start_all_aps(boot_addr); +} + + +/* + * look for the MP spec signature + */ + +/* string defined by the Intel MP Spec as identifying the MP table */ +#define MP_SIG 0x5f504d5f /* _MP_ */ +#define NEXT(X) ((X) += 4) +static int +search_for_sig(u_int32_t target, int count) +{ + int x; + u_int32_t *addr = (u_int32_t *) (KERNBASE + target); + + for (x = 0; x < count; NEXT(x)) + if (addr[x] == MP_SIG) + /* make array index a byte index */ + return (target + (x * sizeof(u_int32_t))); + + return -1; +} + + +#define PROCENTRY_FLAG_EN 0x01 +#define PROCENTRY_FLAG_BP 0x02 +#define IOAPICENTRY_FLAG_EN 0x01 + +/* MP Floating Pointer Structure */ +typedef struct MPFPS { + char signature[4]; + void *pap; + u_char length; + u_char spec_rev; + u_char checksum; + u_char mpfb1; + u_char mpfb2; + u_char mpfb3; + u_char mpfb4; + u_char mpfb5; +} *mpfps_t; +/* MP Configuration Table Header */ +typedef struct MPCTH { + char signature[4]; + u_short base_table_length; + u_char spec_rev; + u_char checksum; + u_char oem_id[8]; + u_char product_id[12]; + void *oem_table_pointer; + u_short oem_table_size; + u_short entry_count; + void *apic_address; + u_short extended_table_length; + u_char extended_table_checksum; + u_char reserved; +} *mpcth_t; + + +typedef struct PROCENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char cpu_flags; + u_long cpu_signature; + u_long feature_flags; + u_long reserved1; + u_long reserved2; +} *proc_entry_ptr; + +typedef struct BUSENTRY { + u_char type; + u_char bus_id; + char bus_type[6]; +} *bus_entry_ptr; + +typedef struct IOAPICENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char apic_flags; + void *apic_address; +} *io_apic_entry_ptr; + +typedef struct INTENTRY { + u_char type; + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} *int_entry_ptr; +/* descriptions of MP basetable entries */ +typedef struct BASETABLE_ENTRY { + u_char type; + u_char length; + char name[16]; +} basetable_entry; + +static basetable_entry basetable_entry_types[] = +{ + {0, 20, "Processor"}, + {1, 8, "Bus"}, + {2, 8, "I/O APIC"}, + {3, 8, "I/O INT"}, + {4, 8, "Local INT"} +}; + +typedef struct BUSDATA { + u_char bus_id; + enum busTypes bus_type; +} bus_datum; + +typedef struct INTDATA { + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} io_int, local_int; + +typedef struct BUSTYPENAME { + u_char type; + char name[7]; +} bus_type_name; + +static bus_type_name bus_type_table[] = +{ + {CBUS, "CBUS"}, + {CBUSII, "CBUSII"}, + {EISA, "EISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {ISA, "ISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {PCI, "PCI"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {XPRESS, "XPRESS"}, + {UNKNOWN_BUSTYPE, "---"} +}; +/* from MP spec v1.4, table 5-1 */ +static int default_data[7][5] = +{ +/* nbus, id0, type0, id1, type1 */ + {1, 0, ISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {0, 255, 255, 255, 255},/* MCA not supported */ + {2, 0, ISA, 1, PCI}, + {2, 0, EISA, 1, PCI}, + {0, 255, 255, 255, 255} /* MCA not supported */ +}; + + +/* the bus data */ +bus_datum bus_data[NBUS]; + +/* the IO INT data, one entry per possible APIC INTerrupt */ +io_int io_apic_ints[NINTR]; + +static int nintrs; + +#if defined(FIX_MP_TABLE_WORKS) +static void fix_mp_table __P((void)); +#endif /* FIX_MP_TABLE_WORKS */ + +static void processor_entry __P((proc_entry_ptr entry, int *cpu)); +static void io_apic_entry __P((io_apic_entry_ptr entry, int *apic)); +static void bus_entry __P((bus_entry_ptr entry, int *bus)); +static void int_entry __P((int_entry_ptr entry, int *intr)); +static int lookup_bus_type __P((char *name)); + + +/* + * parse an Intel MP specification table + */ +static int +parse_mp_table(void) +{ + int x; + mpfps_t fps; + mpcth_t cth; + int totalSize; + void *position; + int count; + int type; + int apic, bus, cpu, intr; + + /* clear physical APIC ID to logical CPU/IO table */ + for (x = 0; x < NAPICID; ++x) + ID_TO_IO(x) = -1; + + /* clear logical CPU to APIC ID table */ + for (x = 0; x < NCPU; ++x) + CPU_TO_ID(x) = -1; + + /* clear logical IO to APIC ID table */ + for (x = 0; x < NAPIC; ++x) + IO_TO_ID(x) = -1; + + /* clear IO APIC address table */ + for (x = 0; x < NAPIC; ++x) + io_apic_address[x] = ~0; + + /* clear bus data table */ + for (x = 0; x < NBUS; ++x) + bus_data[x].bus_id = 0xff; + + /* clear IO APIC INT table */ + for (x = 0; x < NINTR; ++x) + io_apic_ints[x].int_type = 0xff; + nintrs = 0; + + /* count the BSP */ + mp_ncpus = 1; + + /* setup the cpu/apic mapping arrays */ + boot_cpu_id = -1; + + /* local pointer */ + fps = (mpfps_t) mpfps; + + /* record whether PIC or virtual-wire mode */ + picmode = (fps->mpfb2 & 0x80) ? 1 : 0; + + /* check for use of 'default' configuration */ +#if defined(TEST_DEFAULT_CONFIG) + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return TEST_DEFAULT_CONFIG; +#else + if (fps->mpfb1 != 0) { + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return fps->mpfb1; + } +#endif /* TEST_DEFAULT_CONFIG */ + + if ((cth = fps->pap) == 0) + panic("MP Configuration Table Header MISSING!\n"); + + cpu_apic_address = (vm_offset_t) cth->apic_address; + + totalSize = cth->base_table_length - sizeof(struct MPCTH); + position = (u_char *) cth + sizeof(struct MPCTH); + count = cth->entry_count; + + apic = 0; /* logical apic# start @ 0 */ + bus = 0; /* logical bus# start @ 0 */ + cpu = 1; /* logical cpu# start @ 0, BUT reserve 0 for */ + /* BSP */ + intr = 0; /* unknown */ + + /* walk the table, recording info of interest */ + while (count--) { + switch (type = *(u_char *) position) { + case 0: + processor_entry(position, &cpu); + break; + case 1: + bus_entry(position, &bus); + break; + case 2: + io_apic_entry(position, &apic); + break; + case 3: + int_entry(position, &intr); + break; + case 4: + /* int_entry(position); */ + break; + default: + panic("mpfps Base Table HOSED!\n"); + /* NOTREACHED */ + } + + totalSize -= basetable_entry_types[type].length; + (u_char *) position += basetable_entry_types[type].length; + } + + if (boot_cpu_id == -1) + panic("NO BSP found!\n"); + + /* record # of APs found */ + mp_naps = (cpu - 1); + + /* record # of busses found */ + mp_nbusses = bus; + + /* record # of IO APICs found */ + mp_napics = apic; + + /* record # of IO APICs found */ + nintrs = intr; + +#if defined(FIX_MP_TABLE_WORKS) + /* post scan cleanup */ + fix_mp_table(); +#endif /* FIX_MP_TABLE_WORKS */ + + /* report fact that its NOT a default configuration */ + return 0; +} + + +/* + * parse an Intel MP specification table + */ +#if defined(FIX_MP_TABLE_WORKS) +static void +fix_mp_table(void) +{ + int x; + int y; + int num_pci_bus; + bus_datum bus_record; + + /* + * Fix mis-numbering of the PCI bus and its INT entries if the BIOS + * did it wrong. The MP spec says that when more than 1 PCI bus + * exists the BIOS must begin with bus entries for the PCI bus and use + * actual PCI bus numbering. This implies that when only 1 PCI bus + * exists the BIOS can choose to ignore this ordering, and indeed many + * MP motherboards do ignore it. This causes a problem when the PCI + * sub-system makes requests of the MP sub-system based on PCI bus + * numbers. So here we look for the situation and renumber the + * busses and associated INTs in an effort to "make it right". + */ + + /* count the number of PCI busses */ + for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + ++num_pci_bus; + } + + /* check the 1 PCI bus case for sanity */ + if (num_pci_bus == 1) { + + /* if its in the first slot all is well */ + if (bus_data[0].bus_type == PCI) + return; + + /* mis-numbered, swap with whichever bus uses slot 0 */ + + /* locate the entry holding the PCI bus */ + for (x = 1; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + break; + } + + /* swap the bus entry records */ + bus_record = bus_data[0]; + bus_data[0] = bus_data[x]; + bus_data[x] = bus_record; + + /* swap each relavant INTerrupt entry */ + for (y = 0; y < nintrs; ++y) { + if (io_apic_ints[y].src_bus_id == x) + io_apic_ints[y].src_bus_id = 0; + else + if (io_apic_ints[y].src_bus_id == 0) + io_apic_ints[y].src_bus_id = x; + } + } + /* sanity check if more than 1 PCI bus */ + else + if (num_pci_bus > 1) { + for (x = 0; x < num_pci_bus; ++x) { + if (bus_data[x].bus_type != PCI) { + printf("bad PCI bus numbering\n"); + panic("\n"); + } + } + } +} +#endif /* FIX_MP_TABLE_WORKS */ + + +static void +processor_entry(proc_entry_ptr entry, int *cpu) +{ + int x = *cpu; + + /* check for usability */ + if (!(entry->cpu_flags & PROCENTRY_FLAG_EN)) + return; + + /* check for BSP flag */ + if (entry->cpu_flags & PROCENTRY_FLAG_BP) { + /* always give boot CPU the logical value of 0 */ + x = 0; + boot_cpu_id = entry->apic_id; + } else { + /* add another AP to list, if less than max number of CPUs */ + if (x == NCPU) { + printf("Warning: only using %d of the available CPUs!\n", x); + return; + } + ++(*cpu); + } + + CPU_TO_ID(x) = entry->apic_id; + ID_TO_CPU(entry->apic_id) = x; +} + + +static void +bus_entry(bus_entry_ptr entry, int *bus) +{ + int x, y; + char name[8]; + char c; + + if ((x = (*bus)++) == NBUS) + panic("too many busses, increase 'NBUS'\n"); + + /* encode the name into an index */ + for (y = 0; y < 6; ++y) { + if ((c = entry->bus_type[y]) == ' ') + break; + name[y] = c; + } + name[y] = '\0'; + + if ((y = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) + panic("unknown bus type: '%s'\n", name); + + bus_data[x].bus_id = entry->bus_id; + bus_data[x].bus_type = y; +} + + +static void +io_apic_entry(io_apic_entry_ptr entry, int *apic) +{ + int x; + + if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) + return; + + if ((x = (*apic)++) == NAPIC) + panic("too many APICs, increase 'NAPIC'\n"); + + IO_TO_ID(x) = entry->apic_id; + ID_TO_IO(entry->apic_id) = x; + + io_apic_address[x] = (vm_offset_t) entry->apic_address; +} + + +static int +lookup_bus_type(char *name) +{ + int x; + + for (x = 0; x < MAX_BUSTYPE; ++x) + if (strcmp(bus_type_table[x].name, name) == 0) + return bus_type_table[x].type; + + return UNKNOWN_BUSTYPE; +} + + +static void +int_entry(int_entry_ptr entry, int *intr) +{ + int x; + + if ((x = (*intr)++) == NINTR) + panic("too many INTs, increase 'NINTR'\n"); + + io_apic_ints[x].int_type = entry->int_type; + io_apic_ints[x].int_flags = entry->int_flags; + io_apic_ints[x].src_bus_id = entry->src_bus_id; + io_apic_ints[x].src_bus_irq = entry->src_bus_irq; + io_apic_ints[x].dst_apic_id = entry->dst_apic_id; + io_apic_ints[x].dst_apic_int = entry->dst_apic_int; +} + + +static int +apic_int_is_bus_type(int intr, int bus_type) +{ + int bus; + + for (bus = 0; bus < mp_nbusses; ++bus) + if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) + && ((int) bus_data[bus].bus_type == bus_type)) + return 1; + + return 0; +} + + +/* + * determine which APIC pin an ISA INT is attached to. + */ +#define INTTYPE(I) (io_apic_ints[(I)].int_type) +#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) + +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_isa_apic_irq(int isaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (isaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == isaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, ISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin an EISA INT is attached to. + */ +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_eisa_apic_irq(int eisaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (eisaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == eisaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, EISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin a PCI INT is attached to. + */ +#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) +#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) +#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) +int +get_pci_apic_irq(int pciBus, int pciDevice, int pciInt) +{ + int intr; + + --pciInt; /* zero based */ + + for (intr = 0; intr < nintrs; ++intr) /* search each record */ + if ((INTTYPE(intr) == 0) +#if defined(FIX_MP_TABLE_WORKS) + && (SRCBUSID(intr) == pciBus) +#endif /* FIX_MP_TABLE_WORKS */ + && (SRCBUSDEVICE(intr) == pciDevice) + && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, PCI)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSLINE +#undef SRCBUSDEVICE +#undef SRCBUSID + +#undef INTPIN +#undef INTTYPE + + +int +undirect_pci_irq(int rirq) +{ +#if defined(READY) + printf("Freeing irq %d for ISA cards.\n", rirq); + /** FIXME: tickle the MB redirector chip */ + return ???; +#else + printf("Freeing (NOT implimented) irq %d for ISA cards.\n", rirq); + return 0; +#endif /* READY */ +} + + +/* + * given a bus ID, return: + * the bus type if found + * -1 if NOT found + */ +int +apic_bus_type(int id) +{ + int x; + + for (x = 0; x < mp_nbusses; ++x) + if (bus_data[x].bus_id == id) + return bus_data[x].bus_type; + + return -1; +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus ID if found + * -1 if NOT found + */ +int +apic_src_bus_id(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_id); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus IRQ if found + * -1 if NOT found + */ +int +apic_src_bus_irq(int apic, int pin) +{ + int x; + + for (x = 0; x < nintrs; x++) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_irq); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated INTerrupt type if found + * -1 if NOT found + */ +int +apic_int_type(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_type); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated trigger mode if found + * -1 if NOT found + */ +int +apic_trigger(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return ((io_apic_ints[x].int_flags >> 2) & 0x03); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated 'active' level if found + * -1 if NOT found + */ +int +apic_polarity(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_flags & 0x03); + + return -1; /* NOT found */ +} + + +/* + * set data according to MP defaults + * FIXME: probably not complete yet... + */ +static void +default_mp_table(int type) +{ + int ap_cpu_id; +#if defined(APIC_IO) + u_int32_t ux; + int io_apic_id; + int pin; +#endif /* APIC_IO */ + +#if 0 + printf(" MP default config type: %d\n", type); + switch (type) { + case 1: + printf(" bus: ISA, APIC: 82489DX\n"); + break; + case 2: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 3: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 4: + printf(" bus: MCA, APIC: 82489DX\n"); + break; + case 5: + printf(" bus: ISA+PCI, APIC: Integrated\n"); + break; + case 6: + printf(" bus: EISA+PCI, APIC: Integrated\n"); + break; + case 7: + printf(" bus: MCA+PCI, APIC: Integrated\n"); + break; + default: + printf(" future type\n"); + break; + /* NOTREACHED */ + } +#endif /* 0 */ + + boot_cpu_id = (apic_base[APIC_ID] & APIC_ID_MASK) >> 24; + ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; + + /* BSP */ + CPU_TO_ID(0) = boot_cpu_id; + ID_TO_CPU(boot_cpu_id) = 0; + + /* one and only AP */ + CPU_TO_ID(1) = ap_cpu_id; + ID_TO_CPU(ap_cpu_id) = 1; + mp_naps = 1; + + /* one and only IO APIC */ +#if defined(APIC_IO) + io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; + + /* + * sanity check, refer to MP spec section 3.6.6, last paragraph + * necessary as some hardware isn't properly setting up the IO APIC + */ +#if defined(REALLY_ANAL_IOAPICID_VALUE) + if (io_apic_id != 2) { +#else + if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { +#endif /* REALLY_ANAL_IOAPICID_VALUE */ + ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ + ux &= ~APIC_ID_MASK; /* clear the ID field */ + ux |= 0x02000000; /* set it to '2' */ + io_apic_write(0, IOAPIC_ID, ux); /* write new value */ + ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ + if ((ux & APIC_ID_MASK) != 0x02000000) + panic("Problem: can't control IO APIC ID, reg: 0x%08x\n", ux); + io_apic_id = 2; + } + IO_TO_ID(0) = io_apic_id; + ID_TO_IO(io_apic_id) = 0; + mp_napics = 1; +#else + mp_napics = 0; +#endif /* APIC_IO */ + + /* fill out bus entries */ + switch (type) { + case 1: + case 2: + case 3: + case 5: + case 6: + mp_nbusses = default_data[type - 1][0]; + bus_data[0].bus_id = default_data[type - 1][1]; + bus_data[0].bus_type = default_data[type - 1][2]; + bus_data[1].bus_id = default_data[type - 1][3]; + bus_data[1].bus_type = default_data[type - 1][4]; + break; + + /* case 4: case 7: MCA NOT supported */ + default: /* illegal/reserved */ + panic("BAD default MP config: %d\n", type); + } + +#if defined(APIC_IO) + /* general cases from MP v1.4, table 5-2 */ + for (pin = 0; pin < 16; ++pin) { + io_apic_ints[pin].int_type = 0; + io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ + io_apic_ints[pin].src_bus_id = 0; + io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ + io_apic_ints[pin].dst_apic_id = io_apic_id; + io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ + } + + /* special cases from MP v1.4, table 5-2 */ + if (type == 2) { + io_apic_ints[2].int_type = 0xff; /* N/C */ + io_apic_ints[13].int_type = 0xff; /* N/C */ +#if !defined(APIC_MIXED_MODE) + /** FIXME: ??? */ + panic("sorry, can't support type 2 default yet\n"); +#endif /* APIC_MIXED_MODE */ + } else + io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ + + if (type == 7) + io_apic_ints[0].int_type = 0xff; /* N/C */ + else + io_apic_ints[0].int_type = 3; /* vectored 8259 */ + + nintrs = 16; +#endif /* APIC_IO */ +} + + +static void install_ap_tramp(u_int boot_addr); +static int start_ap(int logicalCpu, u_int boot_addr); + +/* + * start each AP in our list + */ +static int +start_all_aps(u_int boot_addr) +{ + int x; + u_char mpbiosreason; + u_long mpbioswarmvec; + + /** + * NOTE: this needs further thought: + * where does it get released? + * should it be set to empy? + * + * get the initial mp_lock with a count of 1 for the BSP + */ + mp_lock = (apic_base[APIC_ID] & APIC_ID_MASK) + 1; + + /* initialize BSP's local APIC */ + apic_initialize(1); + + /* install the AP 1st level boot code */ + install_ap_tramp(boot_addr); + + /* save the current value of the warm-start vector */ + mpbioswarmvec = *((u_long *) WARMBOOT_OFF); + outb(CMOS_REG, BIOS_RESET); + mpbiosreason = inb(CMOS_DATA); + + /* start each AP */ + for (x = 1; x <= mp_naps; ++x) { + + /* setup a vector to our boot code */ + *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; + *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ + + /* attempt to start the Application Processor */ + CHECK_INIT(99); /* setup checkpoints */ + if (!start_ap(x, boot_addr)) { + printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); + CHECK_PRINT("trace"); /* show checkpoints */ + /* + * better panic as the AP may be running loose + * somewhere + */ + printf("panic y/n? [n] "); + if (cngetc() != 'n') + panic("bye-bye\n"); + } + CHECK_PRINT("trace"); /* show checkpoints */ + + /* record its version info */ + cpu_apic_versions[x] = cpu_apic_versions[0]; + } + + /* fill in our (BSP) APIC version */ + cpu_apic_versions[0] = apic_base[APIC_VER]; + + /* restore the warmstart vector */ + *(u_long *) WARMBOOT_OFF = mpbioswarmvec; + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, mpbiosreason); + + /* number of APs actually started */ + return mp_ncpus - 1; +} + + +/* + * load the 1st level AP boot code into base memory. + */ + +/* targets for relocation */ +extern void bigJump(void); +extern void bootCodeSeg(void); +extern void bootDataSeg(void); +extern void MPentry(void); +extern u_int MP_GDT; +extern u_int mp_gdtbase; + +static void +install_ap_tramp(u_int boot_addr) +{ + int x; + int size = *(int *) ((u_long) & bootMP_size); + u_char *src = (u_char *) ((u_long) bootMP); + u_char *dst = (u_char *) boot_addr + KERNBASE; + u_int boot_base = (u_int) bootMP; + u_int8_t *dst8; + u_int16_t *dst16; + u_int32_t *dst32; + + for (x = 0; x < size; ++x) + *dst++ = *src++; + + /* + * modify addresses in code we just moved to basemem. unfortunately we + * need fairly detailed info about mpboot.s for this to work. changes + * to mpboot.s might require changes here. + */ + + /* boot code is located in KERNEL space */ + dst = (u_char *) boot_addr + KERNBASE; + + /* modify the lgdt arg */ + dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); + *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); + + /* modify the ljmp target for MPentry() */ + dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); + *dst32 = ((u_int) MPentry - KERNBASE); + + /* modify the target for boot code segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; + + /* modify the target for boot data segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; +} + + +/* + * this function starts the AP (application processor) identified + * by the APIC ID 'physicalCpu'. It does quite a "song and dance" + * to accomplish this. This is necessary because of the nuances + * of the different hardware we might encounter. It ain't pretty, + * but it seems to work. + */ +static int +start_ap(int logical_cpu, u_int boot_addr) +{ + int physical_cpu; + int vector; + int cpus; + u_long icr_lo, icr_hi; + + /* get the PHYSICAL APIC ID# */ + physical_cpu = CPU_TO_ID(logical_cpu); + + /* calculate the vector */ + vector = (boot_addr >> 12) & 0xff; + + /* used as a watchpoint to signal AP startup */ + cpus = mp_ncpus; + + /* + * first we do an INIT/RESET IPI this INIT IPI might be run, reseting + * and running the target CPU. OR this INIT IPI might be latched (P5 + * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be + * ignored. + */ + + /* setup the address for the target AP */ + icr_hi = apic_base[APIC_ICR_HI] & ~APIC_ID_MASK; + icr_hi |= (physical_cpu << 24); + apic_base[APIC_ICR_HI] = icr_hi; + + /* do an INIT IPI: assert RESET */ + icr_lo = apic_base[APIC_ICR_LOW] & 0xfff00000; + apic_base[APIC_ICR_LOW] = icr_lo | 0x0000c500; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* do an INIT IPI: deassert RESET */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00008500; + + /* wait for pending status end */ + u_sleep(10000); /* wait ~10mS */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* + * next we do a STARTUP IPI: the previous INIT IPI might still be + * latched, (P5 bug) this 1st STARTUP would then terminate + * immediately, and the previously started INIT IPI would continue. OR + * the previous INIT IPI has already run. and this STARTUP IPI will + * run. OR the previous INIT IPI was ignored. and this STARTUP IPI + * will run. + */ + + /* do a STARTUP IPI */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* + * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF + * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR + * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is + * recognized after hardware RESET or INIT IPI. + */ + + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* wait for it to start */ + set_apic_timer(5000000);/* == 5 seconds */ + while (read_apic_timer()) + if (mp_ncpus > cpus) + return 1; /* return SUCCESS */ + + return 0; /* return FAILURE */ +} + + +#if defined(IPI_INTS) + +static void +ipi_intr0(void) +{ + printf("IPI 0\n"); +} + + +static void +ipi_intr1(void) +{ + printf("IPI 1\n"); +} + + +static void +ipi_intr2(void) +{ + printf("IPI 2\n"); +} + + +static void +ipi_intr3(void) +{ + printf("IPI 3\n"); +} + +/*----------------------------------------------------------------------- +** +** Register an interupt handler for an IPI. +** (Stolen from the PCI<->ISA glue code) +** +**----------------------------------------------------------------------- +*/ + +static int +ipi_ihandler_attach(int irq, inthand2_t * func, unsigned *maskptr, int unit) +{ + char buf[16]; + char *cp; + int free_id, id, result; + + sprintf(buf, "ipi irq%d", irq); + for (cp = intrnames, free_id = 0, id = 0; id < NR_DEVICES; id++) { + if (strcmp(cp, buf) == 0) + break; + if (free_id <= 0 && strcmp(cp, "ipi irqnn") == 0) + free_id = id; + while (*cp++ != '\0'); + } + if (id == NR_DEVICES) { + id = free_id; + if (id == 0) { + /* + * All ipi irq counters are in use, perhaps because + * config is old so there aren't any. Abuse the clk0 + * counter. + */ + printf( + "ipi_ihandler_attach: counting ipi irq%d's as clk0 irqs\n", + irq); + } + } + result = register_intr( + irq, /* isa irq */ + id, /* device id */ + 0, /* flags? */ + func, /* handler */ + maskptr, /* mask pointer */ + unit); /* handler arg */ + + if (result) { + printf("WARNING: ipi_ihandler_attach: result=%d\n", result); + return (result); + }; + + return (0); +} +#endif /* IPI_INTS */ + + +#ifdef SMP_INVLTLB +/* + * Flush the TLB on all other CPU's + * + * XXX: Needs to handshake and wait for completion before proceding. + */ + +void +smp_invltlb() +{ + if (smp_active) { + if (invldebug & 2) +#if defined(XFAST_IPI32) + all_but_self_ipi(ICU_OFFSET + 32); +#else + all_but_self_ipi(ICU_OFFSET + 27); +#endif /* XFAST_IPI32 */ + } +} + +void +invlpg(u_int addr) +{ + __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); + smp_invltlb(); +} + +void +invltlb(void) +{ + u_long temp; + /* + * This should be implemented as load_cr3(rcr3()) when load_cr3() is + * inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); + smp_invltlb(); +} +/* + * Handles recieving an "IRQ 27", the invalidate tlb IPI.. + */ +void +ipi_invltlb(void) +{ + u_long temp; + + if (invldebug & 4) { + /* + * This should be implemented as load_cr3(rcr3()) when + * load_cr3() is inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) + :: "memory"); + + } +} +#endif /* SMP_INVLTLB */ diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 26089cc4fdbc..3f218646416e 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -39,7 +39,7 @@ * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $Id: pmap.c,v 1.140 1997/04/13 01:48:08 dyson Exp $ + * $Id: pmap.c,v 1.141 1997/04/13 03:35:30 dyson Exp $ */ /* @@ -68,6 +68,7 @@ * and to when physical maps must be made correct. */ +#include "opt_smp.h" #include "opt_cpu.h" #define PMAP_LOCK 1 @@ -96,10 +97,16 @@ #include <sys/user.h> +#include <machine/cpu.h> #include <machine/pcb.h> #include <machine/cputypes.h> #include <machine/md_var.h> #include <machine/specialreg.h> +#if defined(SMP) || defined(APIC_IO) +#include <machine/smp.h> +#include <machine/apic.h> +#include <machine/smptests.h> +#endif /* SMP || APIC_IO */ #define PMAP_KEEP_PDIRS #define PMAP_SHPGPERPROC 200 @@ -159,7 +166,6 @@ static vm_page_t nkpg; vm_offset_t kernel_vm_end; extern vm_offset_t clean_sva, clean_eva; -extern int cpu_class; #define PV_FREELIST_MIN ((PAGE_SIZE / sizeof (struct pv_entry)) / 2) @@ -182,6 +188,14 @@ static caddr_t CADDR2; static pt_entry_t *msgbufmap; struct msgbuf *msgbufp=0; +#if defined(SMP) || defined(APIC_IO) +static pt_entry_t *apic_map; +#endif /* SMP || APIC_IO */ + +#if defined(APIC_IO) +static pt_entry_t *io_apic_map; +#endif /* APIC_IO */ + pt_entry_t *PMAP1 = 0; unsigned *PADDR1 = 0; @@ -297,6 +311,20 @@ pmap_bootstrap(firstaddr, loadaddr) va = virtual_avail; pte = (pt_entry_t *) pmap_pte(kernel_pmap, va); +#if defined(SMP) || defined(APIC_IO) + /* + * apic_map is the pt for where the local (CPU) apic is mapped in. + */ + SYSMAP(unsigned int *, apic_map, apic_base, 1) +#endif /* SMP || APIC_IO */ + +#if defined(APIC_IO) + /* + * io_apic_map is the pt for where the I/O apic is mapped in. + */ + SYSMAP(unsigned int *, io_apic_map, io_apic_base, 1) +#endif /* APIC_IO */ + /* * CMAP1/CMAP2 are used for zeroing and copying pages. */ @@ -323,14 +351,50 @@ pmap_bootstrap(firstaddr, loadaddr) virtual_avail = va; - *(int *) CMAP1 = *(int *) CMAP2 = *(int *) PTD = 0; + *(int *) CMAP1 = *(int *) CMAP2 = 0; + invltlb(); + +#if !defined(SMP) if (cpu_feature & CPUID_PGE) pgeflag = PG_G; else +#endif /* !SMP */ pgeflag = 0; +} +#if defined(SMP) || defined(APIC_IO) +void +pmap_bootstrap_apics() +{ + if (cpu_apic_address == 0) { + printf("pmap: BSP APIC address NOT found!\n"); + panic("pmap_bootstrap_apics: no apic"); + } + + *(int*)apic_map = PG_V | PG_RW | ((u_long)cpu_apic_address & PG_FRAME); + +#if defined(APIC_IO) +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX +#else + *(int*)io_apic_map = PG_V | PG_RW + | ((u_long)io_apic_address[0] & PG_FRAME); +#endif /* MULTIPLE_IOAPICS */ +#endif /* APIC_IO */ + + invltlb(); } +#endif /* SMP || APIC_IO */ + +#ifdef SMP +void +pmap_bootstrap2() +{ + *(int *) PTD = 0; + invltlb(); +} +#endif /* * Initialize the pmap module. diff --git a/sys/amd64/amd64/support.S b/sys/amd64/amd64/support.S index 0bfef221a9e1..5197f7ea1dc5 100644 --- a/sys/amd64/amd64/support.S +++ b/sys/amd64/amd64/support.S @@ -30,9 +30,10 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: support.s,v 1.50 1997/02/22 09:32:46 peter Exp $ + * $Id: support.s,v 1.51 1997/03/05 16:30:55 bde Exp $ */ +#include "opt_smp.h" #include "npx.h" #include "opt_cpu.h" @@ -40,10 +41,12 @@ #include <machine/cputypes.h> #include <machine/pmap.h> #include <machine/specialreg.h> +#include <machine/smpasm.h> #include "assym.s" #define KDSEL 0x10 /* kernel data selector */ +#define KCSEL 0x8 /* kernel code selector */ #define IDXSHIFT 10 .data @@ -193,7 +196,7 @@ do0: ret #endif -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_bzero) movl 4(%esp),%edx movl 8(%esp),%ecx @@ -436,7 +439,7 @@ ENTRY(generic_bcopy) cld ret -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_bcopy) pushl %esi pushl %edi @@ -621,7 +624,7 @@ ENTRY(copyout) jmp *_copyout_vector ENTRY(generic_copyout) - movl _curpcb,%eax + GETCURPCB(%eax) movl $copyout_fault,PCB_ONFAULT(%eax) pushl %esi pushl %edi @@ -714,7 +717,7 @@ ENTRY(generic_copyout) 3: movl %ebx,%ecx -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ALIGN_TEXT slow_copyout: #endif @@ -732,7 +735,7 @@ done_copyout: popl %edi popl %esi xorl %eax,%eax - movl _curpcb,%edx + GETCURPCB(%edx) movl %eax,PCB_ONFAULT(%edx) ret @@ -741,12 +744,12 @@ copyout_fault: popl %ebx popl %edi popl %esi - movl _curpcb,%edx + GETCURPCB(%edx) movl $0,PCB_ONFAULT(%edx) movl $EFAULT,%eax ret -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_copyout) /* * Duplicated from generic_copyout. Could be done a bit better. @@ -806,7 +809,7 @@ ENTRY(copyin) jmp *_copyin_vector ENTRY(generic_copyin) - movl _curpcb,%eax + GETCURPCB(%eax) movl $copyin_fault,PCB_ONFAULT(%eax) pushl %esi pushl %edi @@ -823,7 +826,7 @@ ENTRY(generic_copyin) cmpl $VM_MAXUSER_ADDRESS,%edx ja copyin_fault -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ALIGN_TEXT slow_copyin: #endif @@ -837,14 +840,14 @@ slow_copyin: rep movsb -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ALIGN_TEXT done_copyin: #endif popl %edi popl %esi xorl %eax,%eax - movl _curpcb,%edx + GETCURPCB(%edx) movl %eax,PCB_ONFAULT(%edx) ret @@ -852,12 +855,12 @@ done_copyin: copyin_fault: popl %edi popl %esi - movl _curpcb,%edx + GETCURPCB(%edx) movl $0,PCB_ONFAULT(%edx) movl $EFAULT,%eax ret -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_copyin) /* * Duplicated from generic_copyin. Could be done a bit better. @@ -892,7 +895,7 @@ ENTRY(i586_copyin) jmp done_copyin #endif /* I586_CPU && NNPX > 0 */ -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) /* fastmove(src, dst, len) src in %esi dst in %edi @@ -1084,7 +1087,7 @@ fastmove_tail_fault: * fu{byte,sword,word} : fetch a byte (sword, word) from user memory */ ENTRY(fuword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx /* from */ @@ -1107,7 +1110,7 @@ ENTRY(fuswintr) ret ENTRY(fusword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1119,7 +1122,7 @@ ENTRY(fusword) ret ENTRY(fubyte) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1132,7 +1135,7 @@ ENTRY(fubyte) ALIGN_TEXT fusufault: - movl _curpcb,%ecx + GETCURPCB(%ecx) xorl %eax,%eax movl %eax,PCB_ONFAULT(%ecx) decl %eax @@ -1142,7 +1145,7 @@ fusufault: * su{byte,sword,word}: write a byte (word, longword) to user memory */ ENTRY(suword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1186,12 +1189,12 @@ ENTRY(suword) movl 8(%esp),%eax movl %eax,(%edx) xorl %eax,%eax - movl _curpcb,%ecx + GETCURPCB(%ecx) movl %eax,PCB_ONFAULT(%ecx) ret ENTRY(susword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1235,13 +1238,13 @@ ENTRY(susword) movw 8(%esp),%ax movw %ax,(%edx) xorl %eax,%eax - movl _curpcb,%ecx /* restore trashed register */ + GETCURPCB(%ecx) /* restore trashed register */ movl %eax,PCB_ONFAULT(%ecx) ret ALTENTRY(suibyte) ENTRY(subyte) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1284,7 +1287,7 @@ ENTRY(subyte) movb 8(%esp),%al movb %al,(%edx) xorl %eax,%eax - movl _curpcb,%ecx /* restore trashed register */ + GETCURPCB(%ecx) /* restore trashed register */ movl %eax,PCB_ONFAULT(%ecx) ret @@ -1298,7 +1301,7 @@ ENTRY(subyte) ENTRY(copyinstr) pushl %esi pushl %edi - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $cpystrflt,PCB_ONFAULT(%ecx) movl 12(%esp),%esi /* %esi = from */ @@ -1346,7 +1349,7 @@ cpystrflt: cpystrflt_x: /* set *lencopied and return %eax */ - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $0,PCB_ONFAULT(%ecx) movl 20(%esp),%ecx subl %edx,%ecx @@ -1446,13 +1449,14 @@ ENTRY(lgdt) movl $KDSEL,%eax movl %ax,%ds movl %ax,%es + movl %ax,%fs + movl %ax,%gs movl %ax,%ss /* reload code selector by turning return into intersegmental return */ movl (%esp),%eax pushl %eax -# movl $KCSEL,4(%esp) - movl $8,4(%esp) + movl $KCSEL,4(%esp) lret /* diff --git a/sys/amd64/amd64/support.s b/sys/amd64/amd64/support.s index 0bfef221a9e1..5197f7ea1dc5 100644 --- a/sys/amd64/amd64/support.s +++ b/sys/amd64/amd64/support.s @@ -30,9 +30,10 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: support.s,v 1.50 1997/02/22 09:32:46 peter Exp $ + * $Id: support.s,v 1.51 1997/03/05 16:30:55 bde Exp $ */ +#include "opt_smp.h" #include "npx.h" #include "opt_cpu.h" @@ -40,10 +41,12 @@ #include <machine/cputypes.h> #include <machine/pmap.h> #include <machine/specialreg.h> +#include <machine/smpasm.h> #include "assym.s" #define KDSEL 0x10 /* kernel data selector */ +#define KCSEL 0x8 /* kernel code selector */ #define IDXSHIFT 10 .data @@ -193,7 +196,7 @@ do0: ret #endif -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_bzero) movl 4(%esp),%edx movl 8(%esp),%ecx @@ -436,7 +439,7 @@ ENTRY(generic_bcopy) cld ret -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_bcopy) pushl %esi pushl %edi @@ -621,7 +624,7 @@ ENTRY(copyout) jmp *_copyout_vector ENTRY(generic_copyout) - movl _curpcb,%eax + GETCURPCB(%eax) movl $copyout_fault,PCB_ONFAULT(%eax) pushl %esi pushl %edi @@ -714,7 +717,7 @@ ENTRY(generic_copyout) 3: movl %ebx,%ecx -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ALIGN_TEXT slow_copyout: #endif @@ -732,7 +735,7 @@ done_copyout: popl %edi popl %esi xorl %eax,%eax - movl _curpcb,%edx + GETCURPCB(%edx) movl %eax,PCB_ONFAULT(%edx) ret @@ -741,12 +744,12 @@ copyout_fault: popl %ebx popl %edi popl %esi - movl _curpcb,%edx + GETCURPCB(%edx) movl $0,PCB_ONFAULT(%edx) movl $EFAULT,%eax ret -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_copyout) /* * Duplicated from generic_copyout. Could be done a bit better. @@ -806,7 +809,7 @@ ENTRY(copyin) jmp *_copyin_vector ENTRY(generic_copyin) - movl _curpcb,%eax + GETCURPCB(%eax) movl $copyin_fault,PCB_ONFAULT(%eax) pushl %esi pushl %edi @@ -823,7 +826,7 @@ ENTRY(generic_copyin) cmpl $VM_MAXUSER_ADDRESS,%edx ja copyin_fault -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ALIGN_TEXT slow_copyin: #endif @@ -837,14 +840,14 @@ slow_copyin: rep movsb -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ALIGN_TEXT done_copyin: #endif popl %edi popl %esi xorl %eax,%eax - movl _curpcb,%edx + GETCURPCB(%edx) movl %eax,PCB_ONFAULT(%edx) ret @@ -852,12 +855,12 @@ done_copyin: copyin_fault: popl %edi popl %esi - movl _curpcb,%edx + GETCURPCB(%edx) movl $0,PCB_ONFAULT(%edx) movl $EFAULT,%eax ret -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_copyin) /* * Duplicated from generic_copyin. Could be done a bit better. @@ -892,7 +895,7 @@ ENTRY(i586_copyin) jmp done_copyin #endif /* I586_CPU && NNPX > 0 */ -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) /* fastmove(src, dst, len) src in %esi dst in %edi @@ -1084,7 +1087,7 @@ fastmove_tail_fault: * fu{byte,sword,word} : fetch a byte (sword, word) from user memory */ ENTRY(fuword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx /* from */ @@ -1107,7 +1110,7 @@ ENTRY(fuswintr) ret ENTRY(fusword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1119,7 +1122,7 @@ ENTRY(fusword) ret ENTRY(fubyte) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1132,7 +1135,7 @@ ENTRY(fubyte) ALIGN_TEXT fusufault: - movl _curpcb,%ecx + GETCURPCB(%ecx) xorl %eax,%eax movl %eax,PCB_ONFAULT(%ecx) decl %eax @@ -1142,7 +1145,7 @@ fusufault: * su{byte,sword,word}: write a byte (word, longword) to user memory */ ENTRY(suword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1186,12 +1189,12 @@ ENTRY(suword) movl 8(%esp),%eax movl %eax,(%edx) xorl %eax,%eax - movl _curpcb,%ecx + GETCURPCB(%ecx) movl %eax,PCB_ONFAULT(%ecx) ret ENTRY(susword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1235,13 +1238,13 @@ ENTRY(susword) movw 8(%esp),%ax movw %ax,(%edx) xorl %eax,%eax - movl _curpcb,%ecx /* restore trashed register */ + GETCURPCB(%ecx) /* restore trashed register */ movl %eax,PCB_ONFAULT(%ecx) ret ALTENTRY(suibyte) ENTRY(subyte) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1284,7 +1287,7 @@ ENTRY(subyte) movb 8(%esp),%al movb %al,(%edx) xorl %eax,%eax - movl _curpcb,%ecx /* restore trashed register */ + GETCURPCB(%ecx) /* restore trashed register */ movl %eax,PCB_ONFAULT(%ecx) ret @@ -1298,7 +1301,7 @@ ENTRY(subyte) ENTRY(copyinstr) pushl %esi pushl %edi - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $cpystrflt,PCB_ONFAULT(%ecx) movl 12(%esp),%esi /* %esi = from */ @@ -1346,7 +1349,7 @@ cpystrflt: cpystrflt_x: /* set *lencopied and return %eax */ - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $0,PCB_ONFAULT(%ecx) movl 20(%esp),%ecx subl %edx,%ecx @@ -1446,13 +1449,14 @@ ENTRY(lgdt) movl $KDSEL,%eax movl %ax,%ds movl %ax,%es + movl %ax,%fs + movl %ax,%gs movl %ax,%ss /* reload code selector by turning return into intersegmental return */ movl (%esp),%eax pushl %eax -# movl $KCSEL,4(%esp) - movl $8,4(%esp) + movl $KCSEL,4(%esp) lret /* diff --git a/sys/amd64/amd64/swtch.s b/sys/amd64/amd64/swtch.s index a74cd09b5a6c..014ddd2f2c57 100644 --- a/sys/amd64/amd64/swtch.s +++ b/sys/amd64/amd64/swtch.s @@ -33,16 +33,24 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: swtch.s,v 1.46 1997/04/20 06:41:26 phk Exp $ + * $Id: swtch.s,v 1.47 1997/04/22 06:55:29 jdp Exp $ */ #include "npx.h" #include "opt_user_ldt.h" +#include "opt_smp.h" +#include "opt_smp_privpages.h" #include <sys/rtprio.h> #include <machine/asmacros.h> #include <machine/spl.h> +#include <machine/smpasm.h> +#include <machine/smptests.h> /** TEST_LOPRIO */ + +#if defined(SMP) && defined(SMP_PRIVPAGES) +#include <machine/pmap.h> +#endif #include "assym.s" @@ -61,8 +69,12 @@ * queues. */ .data +#ifndef SMP .globl _curpcb _curpcb: .long 0 /* pointer to curproc's PCB area */ +#endif + .globl _whichqs, _whichrtqs, _whichidqs + _whichqs: .long 0 /* which run queues have data */ _whichrtqs: .long 0 /* which realtime run queues have data */ _whichidqs: .long 0 /* which idletime run queues have data */ @@ -229,16 +241,30 @@ rem3id: .asciz "remrq.id" /* * When no processes are on the runq, cpu_switch() branches to _idle * to wait for something to come ready. + * + * NOTE: on an SMP system this routine is a startup-only code path. + * once initialization is over, meaning the idle procs have been + * created, we should NEVER branch here. */ ALIGN_TEXT _idle: +#ifdef SMP + movl _smp_active, %eax + cmpl $0, %eax + jnz badsw +#endif /* SMP */ xorl %ebp,%ebp movl $HIDENAME(tmpstk),%esp movl _IdlePTD,%ecx movl %ecx,%cr3 /* update common_tss.tss_esp0 pointer */ +#ifdef SMP + GETCPUID(%eax) + movl _SMPcommon_tss_ptr(,%eax,4), %eax +#else movl $_common_tss, %eax +#endif movl %esp, TSS_ESP0(%eax) #ifdef TSS_IS_CACHED /* example only */ @@ -288,13 +314,20 @@ ENTRY(default_halt) * cpu_switch() */ ENTRY(cpu_switch) + /* switch to new process. first, save context as needed */ - movl _curproc,%ecx + GETCURPROC(%ecx) /* if no process to save, don't bother */ testl %ecx,%ecx je sw1 +#ifdef SMP + movb P_ONCPU(%ecx), %al /* save "last" cpu */ + movb %al, P_LASTCPU(%ecx) + movb $0xff, P_ONCPU(%ecx) /* "leave" the cpu */ +#endif + movl P_ADDR(%ecx),%ecx movl (%esp),%eax /* Hardware registers */ @@ -305,10 +338,19 @@ ENTRY(cpu_switch) movl %esi,PCB_ESI(%ecx) movl %edi,PCB_EDI(%ecx) +#ifdef SMP + movl _mp_lock, %eax + cmpl $0xffffffff, %eax /* is it free? */ + je badsw /* yes, bad medicine! */ + andl $0x00ffffff, %eax /* clear CPU portion */ + movl %eax,PCB_MPNEST(%ecx) /* store it */ +#endif /* SMP */ + #if NNPX > 0 /* have we used fp, and need a save? */ - mov _curproc,%eax - cmp %eax,_npxproc + GETCURPROC(%eax) + GETNPXPROC(%ebx) + cmp %eax,%ebx jne 1f addl $PCB_SAVEFPU,%ecx /* h/w bugs make saving complicated */ pushl %ecx @@ -319,7 +361,7 @@ ENTRY(cpu_switch) movb $1,_intr_nesting_level /* charge Intr, not Sys/Idle */ - movl $0,_curproc /* out of process */ + SETCURPROC($0, %edi) /* save is done, now choose a new process or idle */ sw1: @@ -417,9 +459,30 @@ swtch_com: movl P_ADDR(%ecx),%edx movl PCB_CR3(%edx),%ebx +#if defined(SMP) && defined(SMP_PRIVPAGES) + /* Grab the private PT pointer from the outgoing process's PTD */ + movl $_PTD,%esi + movl 4*MPPTDI(%esi), %eax /* fetch cpu's prv pt */ +#endif + /* switch address space */ movl %ebx,%cr3 +#if defined(SMP) && defined(SMP_PRIVPAGES) + /* Copy the private PT to the new process's PTD */ + /* XXX yuck, the _PTD changes when we switch, so we have to + * reload %cr3 after changing the address space. + * We need to fix this by storing a pointer to the virtual + * location of the per-process PTD in the PCB or something quick. + * Dereferencing proc->vm_map->pmap->p_pdir[] is painful in asm. + */ + movl $_PTD,%esi + movl %eax, 4*MPPTDI(%esi) /* restore cpu's prv page */ + + /* XXX: we have just changed the page tables.. reload.. */ + movl %ebx,%cr3 +#endif + #ifdef HOW_TO_SWITCH_TSS /* example only */ /* Fix up tss pointer to floating pcb/stack structure */ /* XXX probably lots faster to store the 64 bits of tss entry @@ -435,7 +498,12 @@ swtch_com: #endif /* update common_tss.tss_esp0 pointer */ +#ifdef SMP + GETCPUID(%eax) + movl _SMPcommon_tss_ptr(,%eax,4), %eax +#else movl $_common_tss, %eax +#endif movl %edx, %ebx /* pcb */ addl $(UPAGES * PAGE_SIZE), %ebx movl %ebx, TSS_ESP0(%eax) @@ -457,9 +525,32 @@ swtch_com: movl PCB_EIP(%edx),%eax movl %eax,(%esp) - movl %edx,_curpcb - movl %ecx,_curproc /* into next process */ +#ifdef SMP + GETCPUID(%eax) + movb %al, P_ONCPU(%ecx) +#endif + SETCURPCB(%edx, %eax) + SETCURPROC(%ecx, %eax) + movb $0,_intr_nesting_level +#ifdef SMP + movl _apic_base, %eax /* base addr of LOCAL APIC */ +#if defined(TEST_LOPRIO) + /** + * FIXME: the belief here is that we ALWAYS leave here + * holding a the lock, is this TRUE??? + */ + pushl %edx + movl APIC_TPR(%eax), %edx /* get TPR register contents */ + andl $~0xff, %edx /* clear the prio field */ + movl %edx, APIC_TPR(%eax) /* now hold loprio for INTs */ + popl %edx +#endif /* TEST_LOPRIO */ + movl APIC_ID(%eax), %eax /* APIC ID register */ + andl $APIC_ID_MASK, %eax /* extract ID portion */ + orl PCB_MPNEST(%edx), %eax /* add count from PROC */ + movl %eax, _mp_lock /* load the mp_lock */ +#endif /* SMP */ #ifdef USER_LDT cmpl $0, PCB_USERLDT(%edx) @@ -520,7 +611,7 @@ ENTRY(savectx) * have to handle h/w bugs for reloading. We used to lose the * parent's npx state for forks by forgetting to reload. */ - mov _npxproc,%eax + GETNPXPROC(%eax) testl %eax,%eax je 1f diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index 8a86965196d5..aa4e3c5d39d5 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)trap.c 7.4 (Berkeley) 5/13/91 - * $Id: trap.c,v 1.91 1997/04/07 07:15:55 peter Exp $ + * $Id: trap.c,v 1.92 1997/04/14 13:52:52 bde Exp $ */ /* @@ -44,6 +44,7 @@ #include "opt_ktrace.h" #include "opt_ddb.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -76,6 +77,7 @@ #include <machine/reg.h> #include <machine/trap.h> #include <machine/../isa/isa_device.h> +#include <machine/smp.h> #ifdef POWERFAIL_NMI #include <sys/syslog.h> @@ -85,7 +87,11 @@ #include "isa.h" #include "npx.h" +#ifdef SMP +extern struct i386tss *SMPcommon_tss_ptr[NCPU]; +#else extern struct i386tss common_tss; +#endif int (*pmath_emulate) __P((struct trapframe *)); @@ -678,6 +684,9 @@ trap_fatal(frame) printf("\n\nFatal trap %d: %s while in %s mode\n", type, trap_msg[type], ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel"); +#ifdef SMP + printf("cpunumber = %d\n", cpunumber()); +#endif if (type == T_PAGEFLT) { printf("fault virtual address = 0x%x\n", eva); printf("fault code = %s %s, %s\n", @@ -740,6 +749,7 @@ trap_fatal(frame) if (kdb_trap (type, 0, frame)) return; #endif + printf("trap number = %d\n", type); if (type <= MAX_TRAP_MSG) panic(trap_msg[type]); else @@ -761,11 +771,20 @@ trap_fatal(frame) void dblfault_handler() { +#ifdef SMP + int x = cpunumber(); +#endif printf("\nFatal double fault:\n"); +#ifdef SMP + printf("eip = 0x%x\n", SMPcommon_tss_ptr[x]->tss_eip); + printf("esp = 0x%x\n", SMPcommon_tss_ptr[x]->tss_esp); + printf("ebp = 0x%x\n", SMPcommon_tss_ptr[x]->tss_ebp); +#else printf("eip = 0x%x\n", common_tss.tss_eip); printf("esp = 0x%x\n", common_tss.tss_esp); printf("ebp = 0x%x\n", common_tss.tss_ebp); +#endif panic("double fault"); } diff --git a/sys/amd64/amd64/tsc.c b/sys/amd64/amd64/tsc.c index b9425536d67a..d46f982806bd 100644 --- a/sys/amd64/amd64/tsc.c +++ b/sys/amd64/amd64/tsc.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)clock.c 7.2 (Berkeley) 5/12/91 - * $Id: clock.c,v 1.79 1997/03/05 08:08:48 bde Exp $ + * $Id: clock.c,v 1.80 1997/04/06 13:25:48 mckay Exp $ */ /* @@ -49,6 +49,7 @@ */ #include "opt_clock.h" +#include "opt_smp.h" #include "opt_cpu.h" #include <sys/param.h> @@ -100,7 +101,7 @@ int adjkerntz; /* local offset from GMT in seconds */ int disable_rtc_set; /* disable resettodr() if != 0 */ u_int idelayed; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) u_int i586_ctr_bias; u_int i586_ctr_comultiplier; u_int i586_ctr_freq; @@ -143,7 +144,7 @@ static u_char timer0_state; static u_char timer2_state; static void (*timer_func) __P((struct clockframe *frame)) = hardclock; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) static void set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq); #endif static void set_timer_freq(u_int freq, int intr_freq); @@ -553,7 +554,7 @@ calibrate_clocks(void) goto fail; tot_count = 0; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) if (cpu_class == CPUCLASS_586 || cpu_class == CPUCLASS_686) wrmsr(0x10, 0LL); /* XXX 0x10 is the MSR for the TSC */ #endif @@ -586,7 +587,7 @@ calibrate_clocks(void) goto fail; } -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * Read the cpu cycle counter. The timing considerations are * similar to those for the i8254 clock. @@ -667,14 +668,14 @@ startrtclock() printf( "%d Hz differs from default of %d Hz by more than 1%%\n", freq, timer_freq); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) i586_ctr_freq = 0; #endif } set_timer_freq(timer_freq, hz); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) #ifndef CLK_USE_I586_CALIBRATION if (i586_ctr_freq != 0) { if (bootverbose) @@ -821,6 +822,20 @@ resettodr() writertc(RTC_STATUSB, rtc_statusb); } +#if defined(APIC_IO) + +/* from icu.s: */ +extern u_int hwisrs[]; +extern void vec8254 __P((void)); +extern void vecRTC __P((void)); +extern u_int ivectors[]; +extern u_int Xintr8254; +extern u_int XintrRTC; +extern u_int mask8254; +extern u_int maskRTC; + +#endif /* APIC_IO */ + /* * Start both clocks running. */ @@ -828,6 +843,9 @@ void cpu_initclocks() { int diag; +#if defined(APIC_IO) + int x; +#endif /* APIC_IO */ if (statclock_disable) { /* @@ -845,11 +863,47 @@ cpu_initclocks() } /* Finish initializing 8253 timer 0. */ +#if defined(APIC_IO) + /* 8254 is traditionally on ISA IRQ0 */ + if ((x = get_isa_apic_irq(0)) < 0) { + /* + * bummer, this mb doesn't have the 8254 on ISA irq0, + * perhaps it's on the EISA bus... + */ + if ((x = get_eisa_apic_irq(0)) < 0) { + /* double bummer, attempt to redirect thru the 8259 */ + if (bootverbose) + printf("APIC missing 8254 connection\n"); + + /* allow 8254 timer to INTerrupt 8259 */ +#if !defined(IO_ICU1) +#define IO_ICU1 0x20 +#endif + x = inb(IO_ICU1 + 1); /* current mask in 8259 */ + x &= ~1; /* clear 8254 timer mask */ + outb(IO_ICU1 + 1, x); /* write new mask */ + + /* program IO APIC for type 3 INT on INT0 */ + if (ext_int_setup(0, 0) < 0) + panic("8254 redirect impossible!"); + x = 0; /* 8259 is on 0 */ + } + } + + hwisrs[x] = (u_int)vec8254; + Xintr8254 = (u_int)ivectors[x]; /* XXX might need Xfastintr# */ + mask8254 = (1 << x); + register_intr(/* irq */ x, /* XXX id */ 0, /* flags */ 0, + /* XXX */ (inthand2_t *)clkintr, &clk_imask, + /* unit */ 0); + INTREN(mask8254); +#else register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0, /* XXX */ (inthand2_t *)clkintr, &clk_imask, /* unit */ 0); INTREN(IRQ0); -#if defined(I586_CPU) || defined(I686_CPU) +#endif /* APIC_IO */ +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * Finish setting up anti-jitter measures. */ @@ -867,11 +921,36 @@ cpu_initclocks() diag = rtcin(RTC_DIAG); if (diag != 0) printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS); +#if defined(APIC_IO) + /* RTC is traditionally on ISA IRQ8 */ + if ((x = get_isa_apic_irq(8)) < 0) { + if ((x = get_eisa_apic_irq(8)) < 0) { + panic("APIC missing RTC connection"); + } + } + + hwisrs[x] = (u_int)vecRTC; + XintrRTC = (u_int)ivectors[x]; /* XXX might need Xfastintr# */ + maskRTC = (1 << x); + register_intr(/* irq */ x, /* XXX id */ 1, /* flags */ 0, + /* XXX */ (inthand2_t *)rtcintr, &stat_imask, + /* unit */ 0); + INTREN(maskRTC); +#else register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0, /* XXX */ (inthand2_t *)rtcintr, &stat_imask, /* unit */ 0); INTREN(IRQ8); +#endif /* APIC_IO */ writertc(RTC_STATUSB, rtc_statusb); + +#if defined(APIC_IO) + printf("Enabled INTs: "); + for (x = 0; x < 24; ++x) + if ((imen & (1 << x)) == 0) + printf("%d, ", x); + printf("imen: 0x%08x\n", imen); +#endif /* APIC_IO */ } void @@ -900,7 +979,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS if (timer0_state != 0) return (EBUSY); /* too much trouble to handle */ set_timer_freq(freq, hz); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) set_i586_ctr_freq(i586_ctr_freq, timer_freq); #endif } @@ -910,7 +989,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS SYSCTL_PROC(_machdep, OID_AUTO, i8254_freq, CTLTYPE_INT | CTLFLAG_RW, 0, sizeof(u_int), sysctl_machdep_i8254_freq, "I", ""); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) static void set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq) { @@ -949,4 +1028,4 @@ sysctl_machdep_i586_freq SYSCTL_HANDLER_ARGS SYSCTL_PROC(_machdep, OID_AUTO, i586_freq, CTLTYPE_INT | CTLFLAG_RW, 0, sizeof(u_int), sysctl_machdep_i586_freq, "I", ""); -#endif /* defined(I586_CPU) || defined(I686_CPU) */ +#endif /* (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) */ diff --git a/sys/amd64/include/apicreg.h b/sys/amd64/include/apicreg.h new file mode 100644 index 000000000000..a585efba626d --- /dev/null +++ b/sys/amd64/include/apicreg.h @@ -0,0 +1,409 @@ +/* + * Copyright (c) 1996, by Peter Wemm and Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: apic.h,v 1.17 1997/04/26 06:41:36 peter Exp $ + */ + +#ifndef _MACHINE_APIC_H_ +#define _MACHINE_APIC_H_ + +/* + * Local && I/O APIC definitions. + */ + +/* + * Pentium P54C+ Build-in APIC + * (Advanced programmable Interrupt Controller) + * + * Base Address of Build-in APIC in memory location + * is 0xfee00000. + * + * Map of APIC REgisters: + * + * Offset (hex) Description Read/Write state + * 000 Reserved + * 010 Reserved + * 020 ID Local APIC ID R/W + * 030 VER Local APIC Version R + * 040 Reserved + * 050 Reserved + * 060 Reserved + * 070 Reserved + * 080 Task Priority Register R/W + * 090 Arbitration Priority Register R + * 0A0 Processor Priority Register R + * 0B0 EOI Register W + * 0C0 RRR Remote read R + * 0D0 Logical Destination R/W + * 0E0 Destination Format Register 0..27 R; 28..31 R/W + * 0F0 SVR Spurious Interrupt Vector Reg. 0..3 R; 4..9 R/W + * 100 ISR 000-031 R + * 110 ISR 032-063 R + * 120 ISR 064-095 R + * 130 ISR 095-128 R + * 140 ISR 128-159 R + * 150 ISR 160-191 R + * 160 ISR 192-223 R + * 170 ISR 224-255 R + * 180 TMR 000-031 R + * 190 TMR 032-063 R + * 1A0 TMR 064-095 R + * 1B0 TMR 095-128 R + * 1C0 TMR 128-159 R + * 1D0 TMR 160-191 R + * 1E0 TMR 192-223 R + * 1F0 TMR 224-255 R + * 200 IRR 000-031 R + * 210 IRR 032-063 R + * 220 IRR 064-095 R + * 230 IRR 095-128 R + * 240 IRR 128-159 R + * 250 IRR 160-191 R + * 260 IRR 192-223 R + * 270 IRR 224-255 R + * 280 Error Status Register R + * 290 Reserved + * 2A0 Reserved + * 2B0 Reserved + * 2C0 Reserved + * 2D0 Reserved + * 2E0 Reserved + * 2F0 Reserved + * 300 ICR_LOW Interrupt Command Reg. (0-31) R/W + * 310 ICR_HI Interrupt Command Reg. (32-63) R/W + * 320 Local Vector Table (Timer) R/W + * 330 Reserved + * 340 Reserved + * 350 LVT1 Local Vector Table (LINT0) R/W + * 360 LVT2 Local Vector Table (LINT1) R/W + * 370 LVT3 Local Vector Table (ERROR) R/W + * 380 Initial Count Reg. for Timer R/W + * 390 Current Count of Timer R + * 3A0 Reserved + * 3B0 Reserved + * 3C0 Reserved + * 3D0 Reserved + * 3E0 Timer Divide Configuration Reg. R/W + * 3F0 Reserved + */ + + +/****************************************************************************** + * global defines, etc. + */ + +/* enable the InterProcessor Interrupt code, FIXME: temporary marker */ +#define IPI_INTS + +/** + * this enables code concerned with handling more than one IO APIC. + * Note: this is NOT READY for use! + * +#define MULTIPLE_IOAPICS + */ + +/****************************************************************************** + * LOCAL APIC defines + */ + +/* default physical locations of LOCAL (CPU) APICs */ +#define DEFAULT_APIC_BASE 0xfee00000 + +# if defined(LOCORE) + +#define APIC_ID 0x020 +#define APIC_VER 0x030 +#define APIC_TPR 0x080 +#define APIC_APR 0x090 +#define APIC_PPR 0x0a0 +#define APIC_EOI 0x0b0 +#define APIC_RR 0x0c0 +#define APIC_LDR 0x0d0 +#define APIC_DFR 0x0e0 +#define APIC_SVR 0x0f0 +#define APIC_ISR 0x100 +#define APIC_ISR0 0x100 +#define APIC_ISR1 0x110 +#define APIC_ISR2 0x120 +#define APIC_TMR 0x180 +#define APIC_IRR 0x200 +#define APIC_IRR0 0x200 +#define APIC_IRR1 0x210 +#define APIC_IRR2 0x220 +#define APIC_ESR 0x280 +#define APIC_ICR_LOW 0x300 +#define APIC_ICR_HI 0x310 +#define APIC_LVTT 0x320 +#define APIC_LVT1 0x350 +#define APIC_LVT2 0x360 +#define APIC_LVT3 0x370 +#define APIC_TICR 0x380 +#define APIC_TCCR 0x390 +#define APIC_TDCR 0x3e0 + +# else /* !LOCORE */ + +/* offsets in apic_base[] */ +#define APIC_ID (0x020/4) +#define APIC_VER (0x030/4) +#define APIC_TPR (0x080/4) +#define APIC_APR (0x090/4) +#define APIC_PPR (0x0a0/4) +#define APIC_EOI (0x0b0/4) +#define APIC_RR (0x0c0/4) +#define APIC_LDR (0x0d0/4) +#define APIC_DFR (0x0e0/4) +#define APIC_SVR (0x0f0/4) +#define APIC_ISR (0x100/4) +#define APIC_ISR0 (0x100/4) +#define APIC_ISR1 (0x110/4) +#define APIC_ISR2 (0x120/4) +#define APIC_TMR (0x180/4) +#define APIC_IRR (0x200/4) +#define APIC_IRR0 (0x200/4) +#define APIC_IRR1 (0x210/4) +#define APIC_IRR2 (0x220/4) +#define APIC_ESR (0x280/4) +#define APIC_ICR_LOW (0x300/4) +#define APIC_ICR_HI (0x310/4) +#define APIC_LVTT (0x320/4) +#define APIC_LVT1 (0x350/4) +#define APIC_LVT2 (0x360/4) +#define APIC_LVT3 (0x370/4) +#define APIC_TICR (0x380/4) +#define APIC_TCCR (0x390/4) +#define APIC_TDCR (0x3e0/4) + +# endif /* LOCORE */ + + +/* fields in VER */ +#define APIC_VER_VERSION 0x000000ff +#define APIC_VER_MAXLVT 0x00ff0000 +#define MAXLVTSHIFT 16 + +/* fields in SVR */ +#define APIC_SVR_ENABLE 0x00000100 +# define APIC_SVR_SWDIS 0x00000000 +# define APIC_SVR_SWEN 0x00000100 +#define APIC_SVR_FOCUS 0x00000200 +# define APIC_SVR_FEN 0x00000000 +# define APIC_SVR_FDIS 0x00000200 +#define APIC_TPR_PRIO 0x000000ff +# define APIC_TPR_INT 0x000000f0 +# define APIC_TPR_SUB 0x0000000f + + +/* fields in ICR_LOW */ +#define APIC_VECTOR_MASK 0x000000ff + +#define APIC_DELMODE_MASK 0x00000700 +# define APIC_DELMODE_FIXED 0x00000000 +# define APIC_DELMODE_LOWPRIO 0x00000100 +# define APIC_DELMODE_SMI 0x00000200 +# define APIC_DELMODE_RR 0x00000300 +# define APIC_DELMODE_NMI 0x00000400 +# define APIC_DELMODE_INIT 0x00000500 +# define APIC_DELMODE_STARTUP 0x00000600 +# define APIC_DELMODE_RESV 0x00000700 + +#define APIC_DESTMODE_MASK 0x00000800 +# define APIC_DESTMODE_PHY 0x00000000 +# define APIC_DESTMODE_LOG 0x00000800 + +#define APIC_DELSTAT_MASK 0x00001000 +# define APIC_DELSTAT_IDLE 0x00000000 +# define APIC_DELSTAT_PEND 0x00001000 + +#define APIC_RESV1_MASK 0x00002000 + +#define APIC_LEVEL_MASK 0x00004000 +# define APIC_LEVEL_DEASSERT 0x00000000 +# define APIC_LEVEL_ASSERT 0x00004000 + +#define APIC_TRIGMOD_MASK 0x00008000 +# define APIC_TRIGMOD_EDGE 0x00000000 +# define APIC_TRIGMOD_LEVEL 0x00008000 + +#define APIC_RRSTAT_MASK 0x00030000 +# define APIC_RRSTAT_INVALID 0x00000000 +# define APIC_RRSTAT_INPROG 0x00010000 +# define APIC_RRSTAT_VALID 0x00020000 +# define APIC_RRSTAT_RESV 0x00030000 + +#define APIC_DEST_MASK 0x000c0000 +# define APIC_DEST_DESTFLD 0x00000000 +# define APIC_DEST_SELF 0x00040000 +# define APIC_DEST_ALLISELF 0x00080000 +# define APIC_DEST_ALLESELF 0x000c0000 + +#define APIC_RESV2_MASK 0xfff00000 + + +/* fields in ICR_HIGH */ +#define APIC_ID_MASK 0x0f000000 + + +/* fields in LVT1/2 */ +#define APIC_LVT_VECTOR 0x000000ff +#define APIC_LVT_DS 0x00001000 +#define APIC_LVT_M 0x00010000 + + +/* fields in LVT Timer */ +#define APIC_LVTT_VECTOR 0x000000ff +#define APIC_LVTT_DS 0x00001000 +#define APIC_LVTT_M 0x00010000 +#define APIC_LVTT_TM 0x00020000 + + +/* fields in TDCR */ +#define APIC_TDCR_2 0x00 +#define APIC_TDCR_4 0x01 +#define APIC_TDCR_8 0x02 +#define APIC_TDCR_16 0x03 +#define APIC_TDCR_32 0x08 +#define APIC_TDCR_64 0x09 +#define APIC_TDCR_128 0x0a +#define APIC_TDCR_1 0x0b + + +/* + * fields in IRR + * ISA INTerrupts are in bits 16-31 of the 1st IRR register. + * these masks DON'T EQUAL the isa IRQs of the same name. + * FIXME: how do we make this portable for MP table configurations??? + * look for "HARD_VECTORXXX" marking places with this problem. + */ +#define APIC_IRQ0 0x00000001 +#define APIC_IRQ1 0x00000002 +#define APIC_IRQ2 0x00000004 +#define APIC_IRQ3 0x00000008 +#define APIC_IRQ4 0x00000010 +#define APIC_IRQ5 0x00000020 +#define APIC_IRQ6 0x00000040 +#define APIC_IRQ7 0x00000080 +#define APIC_IRQ8 0x00000100 +#define APIC_IRQ9 0x00000200 +#define APIC_IRQ10 0x00000400 +#define APIC_IRQ11 0x00000800 +#define APIC_IRQ12 0x00001000 +#define APIC_IRQ13 0x00002000 +#define APIC_IRQ14 0x00004000 +#define APIC_IRQ15 0x00008000 +#define APIC_IRQ16 0x00010000 +#define APIC_IRQ17 0x00020000 +#define APIC_IRQ18 0x00040000 +#define APIC_IRQ19 0x00080000 +#define APIC_IRQ20 0x00100000 +#define APIC_IRQ21 0x00200000 +#define APIC_IRQ22 0x00400000 +#define APIC_IRQ23 0x00800000 + + +/****************************************************************************** + * I/O APIC defines + */ + +/* default physical locations of an IO APIC */ +#define DEFAULT_IO_APIC_BASE 0xfec00000 + +/* window register offset */ +#define IOAPIC_WINDOW 0x10 + +/* indexes into IO APIC */ +#define IOAPIC_ID 0x00 +#define IOAPIC_VER 0x01 +#define IOAPIC_ARB 0x02 +#define IOAPIC_REDTBL 0x10 +#define IOAPIC_REDTBL0 IOAPIC_REDTBL +#define IOAPIC_REDTBL1 (IOAPIC_REDTBL+0x02) +#define IOAPIC_REDTBL2 (IOAPIC_REDTBL+0x04) +#define IOAPIC_REDTBL3 (IOAPIC_REDTBL+0x06) +#define IOAPIC_REDTBL4 (IOAPIC_REDTBL+0x08) +#define IOAPIC_REDTBL5 (IOAPIC_REDTBL+0x0a) +#define IOAPIC_REDTBL6 (IOAPIC_REDTBL+0x0c) +#define IOAPIC_REDTBL7 (IOAPIC_REDTBL+0x0e) +#define IOAPIC_REDTBL8 (IOAPIC_REDTBL+0x10) +#define IOAPIC_REDTBL9 (IOAPIC_REDTBL+0x12) +#define IOAPIC_REDTBL10 (IOAPIC_REDTBL+0x14) +#define IOAPIC_REDTBL11 (IOAPIC_REDTBL+0x16) +#define IOAPIC_REDTBL12 (IOAPIC_REDTBL+0x18) +#define IOAPIC_REDTBL13 (IOAPIC_REDTBL+0x1a) +#define IOAPIC_REDTBL14 (IOAPIC_REDTBL+0x1c) +#define IOAPIC_REDTBL15 (IOAPIC_REDTBL+0x1e) +#define IOAPIC_REDTBL16 (IOAPIC_REDTBL+0x20) +#define IOAPIC_REDTBL17 (IOAPIC_REDTBL+0x22) +#define IOAPIC_REDTBL18 (IOAPIC_REDTBL+0x24) +#define IOAPIC_REDTBL19 (IOAPIC_REDTBL+0x26) +#define IOAPIC_REDTBL20 (IOAPIC_REDTBL+0x28) +#define IOAPIC_REDTBL21 (IOAPIC_REDTBL+0x2a) +#define IOAPIC_REDTBL22 (IOAPIC_REDTBL+0x2c) +#define IOAPIC_REDTBL23 (IOAPIC_REDTBL+0x2e) + +/* fields in VER */ +#define IOART_VER_VERSION 0x000000ff +#define IOART_VER_MAXREDIR 0x00ff0000 +#define MAXREDIRSHIFT 16 + +/* + * fields in the IO APIC's redirection table entries + */ +#define IOART_DEST APIC_ID_MASK /* broadcast addr: all APICs */ + +#define IOART_RESV 0x00fe0000 /* reserved */ + +#define IOART_INTMASK 0x00010000 /* R/W: INTerrupt mask */ +# define IOART_INTMCLR 0x00000000 /* clear, allow INTs */ +# define IOART_INTMSET 0x00010000 /* set, inhibit INTs */ + +#define IOART_TRGRMOD 0x00008000 /* R/W: trigger mode */ +# define IOART_TRGREDG 0x00000000 /* edge */ +# define IOART_TRGRLVL 0x00008000 /* level */ + +#define IOART_REM_IRR 0x00004000 /* RO: remote IRR */ + +#define IOART_INTPOL 0x00002000 /* R/W: INT input pin polarity */ +# define IOART_INTAHI 0x00000000 /* active high */ +# define IOART_INTALO 0x00002000 /* active low */ + +#define IOART_DELIVS 0x00001000 /* RO: delivery status */ + +#define IOART_DESTMOD 0x00000800 /* R/W: destination mode */ +# define IOART_DESTPHY 0x00000000 /* physical */ +# define IOART_DESTLOG 0x00000800 /* logical */ + +#define IOART_DELMOD 0x00000700 /* R/W: delivery mode */ +# define IOART_DELFIXED 0x00000000 /* fixed */ +# define IOART_DELLOPRI 0x00000100 /* lowest priority */ +# define IOART_DELSMI 0x00000200 /* System Management INT */ +# define IOART_DELRSV1 0x00000300 /* reserved */ +# define IOART_DELNMI 0x00000400 /* NMI signal */ +# define IOART_DELINIT 0x00000500 /* INIT signal */ +# define IOART_DELRSV2 0x00000600 /* reserved */ +# define IOART_DELEXINT 0x00000700 /* External INTerrupt */ + +#define IOART_INTVEC 0x000000ff /* R/W: INTerrupt vector field */ + +#endif /* _MACHINE_APIC_H_ */ diff --git a/sys/amd64/include/clock.h b/sys/amd64/include/clock.h index 30ff97a7f80f..df8a20c5e4e8 100644 --- a/sys/amd64/include/clock.h +++ b/sys/amd64/include/clock.h @@ -3,13 +3,13 @@ * Garrett Wollman, September 1994. * This file is in the public domain. * - * $Id$ + * $Id: clock.h,v 1.24 1997/02/22 09:33:59 peter Exp $ */ #ifndef _MACHINE_CLOCK_H_ #define _MACHINE_CLOCK_H_ -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) #define CPU_CLOCKUPDATE(otime, ntime) cpu_clockupdate((otime), (ntime)) #else #define CPU_CLOCKUPDATE(otime, ntime) (*(otime) = *(ntime)) @@ -28,7 +28,7 @@ */ extern int adjkerntz; extern int disable_rtc_set; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) extern u_int i586_ctr_bias; extern u_int i586_ctr_comultiplier; extern u_int i586_ctr_freq; @@ -81,7 +81,7 @@ clock_latency(void) - ((high << 8) | low)); } -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * When we update `time', on i586's we also update `i586_ctr_bias' * atomically. `i586_ctr_bias' is the best available approximation to diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h index 9cd03517e3a1..740b7b23765f 100644 --- a/sys/amd64/include/cpufunc.h +++ b/sys/amd64/include/cpufunc.h @@ -30,7 +30,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: cpufunc.h,v 1.61 1997/02/22 09:34:08 peter Exp $ + * $Id: cpufunc.h,v 1.62 1997/03/22 18:52:57 kato Exp $ */ /* @@ -42,6 +42,10 @@ #include <sys/cdefs.h> #include <sys/types.h> +#include <machine/smp.h> + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" #ifdef __GNUC__ @@ -201,6 +205,16 @@ invd(void) __asm __volatile("invd"); } +#if defined(SMP) && defined(SMP_INVLTLB) + +/* + * When using APIC IPI's, the inlining cost is prohibitive.. + */ +void invlpg __P((u_int addr)); +void invltlb __P((void)); + +#else + static __inline void invlpg(u_int addr) { @@ -218,6 +232,7 @@ invltlb(void) __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) : : "memory"); } +#endif /* SMP && SMP_INVLTLB */ static __inline u_short inw(u_int port) diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h new file mode 100644 index 000000000000..07fca34efe56 --- /dev/null +++ b/sys/amd64/include/mptable.h @@ -0,0 +1,1646 @@ +/* + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: mp_machdep.c,v 1.47 1997/04/26 08:11:49 peter Exp $ + */ + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" +#if defined(APIC_IO) +#if !defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#else /* APIC_IO */ +#if defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#endif /* APIC_IO */ +#define FIX_MP_TABLE_WORKS_NOT + +#include "opt_serial.h" + +#include <sys/param.h> /* for KERNBASE */ +#include <sys/types.h> +#include <sys/sysproto.h> +#include <sys/time.h> +#include <sys/systm.h> + +#include <vm/vm.h> /* for KERNBASE */ +#include <vm/vm_param.h> /* for KERNBASE */ +#include <vm/pmap.h> /* for KERNBASE */ +#include <machine/pmap.h> /* for KERNBASE */ + +#include <machine/smp.h> +#include <machine/apic.h> +#include <machine/mpapic.h> +#include <machine/cpufunc.h> +#include <machine/segments.h> +#include <machine/smptests.h> /** TEST_UPPERPRIO, TEST_DEFAULT_CONFIG */ + +#include <i386/i386/cons.h> /* cngetc() */ + +#if defined(IPI_INTS) +#include <i386/isa/isa_device.h> +#include "vector.h" +#endif /* IPI_INTS */ + +#if defined(SMP_INVLTLB) +#include <i386/isa/icu.h> +#endif /* SMP_INVLTLB */ + +#define WARMBOOT_TARGET 0 +#define WARMBOOT_OFF (KERNBASE + 0x0467) +#define WARMBOOT_SEG (KERNBASE + 0x0469) + +#define BIOS_BASE (0xf0000) +#define BIOS_SIZE (0x10000) +#define BIOS_COUNT (BIOS_SIZE/4) + +#define CMOS_REG (0x70) +#define CMOS_DATA (0x71) +#define BIOS_RESET (0x0f) +#define BIOS_WARM (0x0a) + +/* + * this code MUST be enabled here and in mpboot.s. + * it follows the very early stages of AP boot by placing values in CMOS ram. + * it NORMALLY will never be needed and thus the primitive method for enabling. + * +#define CHECK_POINTS + */ + +#if defined(CHECK_POINTS) +#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) +#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) + +#define CHECK_INIT(D); \ + CHECK_WRITE(0x34, (D)); \ + CHECK_WRITE(0x35, (D)); \ + CHECK_WRITE(0x36, (D)); \ + CHECK_WRITE(0x37, (D)); \ + CHECK_WRITE(0x38, (D)); \ + CHECK_WRITE(0x39, (D)); + +#define CHECK_PRINT(S); \ + printf("%s: %d, %d, %d, %d, %d, %d\n", \ + (S), \ + CHECK_READ(0x34), \ + CHECK_READ(0x35), \ + CHECK_READ(0x36), \ + CHECK_READ(0x37), \ + CHECK_READ(0x38), \ + CHECK_READ(0x39)); + +#else /* CHECK_POINTS */ + +#define CHECK_INIT(D) +#define CHECK_PRINT(S) + +#endif /* CHECK_POINTS */ + + +/** FIXME: what system files declare these??? */ +extern struct region_descriptor r_gdt, r_idt; + +/* global data */ +struct proc *SMPcurproc[NCPU]; +struct pcb *SMPcurpcb[NCPU]; +struct timeval SMPruntime[NCPU]; + +int mp_ncpus; /* # of CPUs, including BSP */ +int mp_naps; /* # of Applications processors */ +int mp_nbusses; /* # of busses */ +int mp_napics; /* # of IO APICs */ +int mpenabled; +int boot_cpu_id; /* designated BSP */ +vm_offset_t cpu_apic_address; +vm_offset_t io_apic_address[NAPIC]; + +u_int32_t cpu_apic_versions[NCPU]; +u_int32_t io_apic_versions[NAPIC]; + +/* + * APIC ID logical/physical mapping structures + */ +int cpu_num_to_apic_id[NCPU]; +int io_num_to_apic_id[NAPIC]; +int apic_id_to_logical[NAPICID]; + +/* + * look for MP compliant motherboard. + */ + +static u_int boot_address; +static u_int base_memory; + +static int picmode; /* 0: virtual wire mode, 1: PIC mode */ +static u_int mpfps; +static int search_for_sig(u_int32_t target, int count); +static int mp_probe(u_int base_top); +static void mp_enable(u_int boot_addr); + +#if defined(IPI_INTS) +static void ipi_intr0(void); +static void ipi_intr1(void); +static void ipi_intr2(void); +static void ipi_intr3(void); +static int +ipi_ihandler_attach(int irq, inthand2_t * func, + unsigned *maskptr, int unit); +#endif /* IPI_INTS */ + + +/* + * calculate usable address in base memory for AP trampoline code + */ +u_int +mp_bootaddress(u_int basemem) +{ + base_memory = basemem * 1024; /* convert to bytes */ + + boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ + if ((base_memory - boot_address) < bootMP_size) + boot_address -= 4096; /* not enough, lower by 4k */ + + return boot_address; +} + + +/* + * startup the SMP processors + */ +void +mp_start(void) +{ + /* look for MP capable motherboard */ + if (mp_probe(base_memory)) + mp_enable(boot_address); + else { + printf("MP FPS NOT FOUND, suggest use of 'mptable' program\n"); + panic("can't continue!\n"); + } + + /* finish pmap initialization - turn off V==P mapping at zero */ + pmap_bootstrap2(); +} + + +/* + * print various information about the SMP system hardware and setup + */ +void +mp_announce(void) +{ + int x; + + printf("FreeBSD/SMP: Multiprocessor motherboard\n"); + printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); + printf(", version: 0x%08x\n", cpu_apic_versions[0]); + for (x = 1; x <= mp_naps; ++x) { + printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); + printf(", version: 0x%08x\n", cpu_apic_versions[x]); + } + +#if defined(APIC_IO) + for (x = 0; x < mp_napics; ++x) { + printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); + printf(", version: 0x%08x\n", io_apic_versions[x]); + } +#else + printf(" Warning: APIC I/O disabled\n"); +#endif /* APIC_IO */ +} + + +/* + * AP cpu's call this to sync up protected mode. + */ +void +init_secondary(void) +{ + int gsel_tss, slot; + + r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; + r_gdt.rd_base = (int) gdt; + lgdt(&r_gdt); /* does magic intra-segment return */ + lidt(&r_idt); + lldt(_default_ldt); + + slot = NGDT + cpunumber(); + gsel_tss = GSEL(slot, SEL_KPL); + gdt[slot].sd.sd_type = SDT_SYS386TSS; + ltr(gsel_tss); + + load_cr0(0x8005003b); /* XXX! */ +} + + +#if defined(APIC_IO) +void +configure_local_apic(void) +{ + u_char byte; + u_int32_t temp; + + if (picmode) { + outb(0x22, 0x70); /* select IMCR */ + byte = inb(0x23); /* current contents */ + byte |= 0x01; /* mask external INTR */ + outb(0x23, byte); /* disconnect 8259s/NMI */ + } + /* mask the LVT1 */ + temp = apic_base[APIC_LVT1]; + temp |= APIC_LVT_M; + apic_base[APIC_LVT1] = temp; +} +#endif /* APIC_IO */ + + +/******************************************************************* + * local functions and data + */ + +static int +mp_probe(u_int base_top) +{ + int x; + u_long segment; + u_int32_t target; + + /* see if EBDA exists */ + if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { + /* search first 1K of EBDA */ + target = (u_int32_t) (segment << 4); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } else { + /*last 1K of base memory, effective 'top of base' is passed in*/ + target = (u_int32_t) (base_top - 0x400); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } + + /* search the BIOS */ + target = (u_int32_t) BIOS_BASE; + if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) + goto found; + + /* nothing found */ + mpfps = mpenabled = 0; + return 0; + +found: /* please forgive the 'goto'! */ + /* flag fact that we are running multiple processors */ + mpfps = x; + mpenabled = 1; + return 1; +} + + +/* + * start the SMP system + */ +static int parse_mp_table(void); +static void default_mp_table(int type); +static int start_all_aps(u_int boot_addr); + +#if defined(XFAST_IPI32) +#include <machine/md_var.h> +#include <i386/isa/isa_device.h> +extern void Xfastipi32(u_int, u_int, u_int, u_int); +#endif /* XFAST_IPI32 */ + +static void +mp_enable(u_int boot_addr) +{ + int x; +#if defined(APIC_IO) + int apic; + u_int ux; +#if defined(TEST_UPPERPRIO) + u_char select; /* the select register is 8 bits */ + u_int32_t flags; /* the window register is 32 bits */ +#endif /* TEST_UPPERPRIO */ +#endif /* APIC_IO */ + + /* examine the MP table for needed info */ + x = parse_mp_table(); + + /* create pages for (address common) cpu APIC and each IO APIC */ + pmap_bootstrap_apics(); + + /* can't process default configs till the CPU APIC is pmapped */ + if (x) + default_mp_table(x); + +#if defined(APIC_IO) + /* fill the LOGICAL io_apic_versions table */ + for (apic = 0; apic < mp_napics; ++apic) { + ux = io_apic_read(apic, IOAPIC_VER); + io_apic_versions[apic] = ux; + } + + /* + */ + for (apic = 0; apic < mp_napics; ++apic) + if (io_apic_setup(apic) < 0) + panic("IO APIC setup failure\n"); + +#if defined(IPI_INTS) + /* setup IPI INTerrupt mechanism */ + ipi_ihandler_attach( /* irq */ 24, + /* XXX */ (inthand2_t *) ipi_intr0, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 25, + /* XXX */ (inthand2_t *) ipi_intr1, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 26, + /* XXX */ (inthand2_t *) ipi_intr2, NULL, /* unit */ 0); +#if !defined(SMP_INVLTLB) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); +#else +#if defined(XFAST_IPI32) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); + setidt(ICU_OFFSET + 32, + Xfastipi32, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#else + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_invltlb, NULL, /* unit */ 0); +#endif /* XFAST_IPI32 */ +#endif + +#endif /* IPI_INTS */ + +#if defined(TEST_UPPERPRIO) + +#if 1 + printf("special IRQ10\n"); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~0xff; /** clear vector */ + flags |= 64; + io_apic_write(0, select, flags); +#else + printf("special IRQ10\n"); + cngetc(); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~IOART_DELMOD; /* FIXED mode */ + io_apic_write(0, select, flags); + io_apic_write(0, select + 1, boot_cpu_id << 24); +#endif /** 0/1 */ + +#endif /* TEST_UPPERPRIO */ + +#endif /* APIC_IO */ + + /* start each Application Processor */ + start_all_aps(boot_addr); +} + + +/* + * look for the MP spec signature + */ + +/* string defined by the Intel MP Spec as identifying the MP table */ +#define MP_SIG 0x5f504d5f /* _MP_ */ +#define NEXT(X) ((X) += 4) +static int +search_for_sig(u_int32_t target, int count) +{ + int x; + u_int32_t *addr = (u_int32_t *) (KERNBASE + target); + + for (x = 0; x < count; NEXT(x)) + if (addr[x] == MP_SIG) + /* make array index a byte index */ + return (target + (x * sizeof(u_int32_t))); + + return -1; +} + + +#define PROCENTRY_FLAG_EN 0x01 +#define PROCENTRY_FLAG_BP 0x02 +#define IOAPICENTRY_FLAG_EN 0x01 + +/* MP Floating Pointer Structure */ +typedef struct MPFPS { + char signature[4]; + void *pap; + u_char length; + u_char spec_rev; + u_char checksum; + u_char mpfb1; + u_char mpfb2; + u_char mpfb3; + u_char mpfb4; + u_char mpfb5; +} *mpfps_t; +/* MP Configuration Table Header */ +typedef struct MPCTH { + char signature[4]; + u_short base_table_length; + u_char spec_rev; + u_char checksum; + u_char oem_id[8]; + u_char product_id[12]; + void *oem_table_pointer; + u_short oem_table_size; + u_short entry_count; + void *apic_address; + u_short extended_table_length; + u_char extended_table_checksum; + u_char reserved; +} *mpcth_t; + + +typedef struct PROCENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char cpu_flags; + u_long cpu_signature; + u_long feature_flags; + u_long reserved1; + u_long reserved2; +} *proc_entry_ptr; + +typedef struct BUSENTRY { + u_char type; + u_char bus_id; + char bus_type[6]; +} *bus_entry_ptr; + +typedef struct IOAPICENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char apic_flags; + void *apic_address; +} *io_apic_entry_ptr; + +typedef struct INTENTRY { + u_char type; + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} *int_entry_ptr; +/* descriptions of MP basetable entries */ +typedef struct BASETABLE_ENTRY { + u_char type; + u_char length; + char name[16]; +} basetable_entry; + +static basetable_entry basetable_entry_types[] = +{ + {0, 20, "Processor"}, + {1, 8, "Bus"}, + {2, 8, "I/O APIC"}, + {3, 8, "I/O INT"}, + {4, 8, "Local INT"} +}; + +typedef struct BUSDATA { + u_char bus_id; + enum busTypes bus_type; +} bus_datum; + +typedef struct INTDATA { + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} io_int, local_int; + +typedef struct BUSTYPENAME { + u_char type; + char name[7]; +} bus_type_name; + +static bus_type_name bus_type_table[] = +{ + {CBUS, "CBUS"}, + {CBUSII, "CBUSII"}, + {EISA, "EISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {ISA, "ISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {PCI, "PCI"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {XPRESS, "XPRESS"}, + {UNKNOWN_BUSTYPE, "---"} +}; +/* from MP spec v1.4, table 5-1 */ +static int default_data[7][5] = +{ +/* nbus, id0, type0, id1, type1 */ + {1, 0, ISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {0, 255, 255, 255, 255},/* MCA not supported */ + {2, 0, ISA, 1, PCI}, + {2, 0, EISA, 1, PCI}, + {0, 255, 255, 255, 255} /* MCA not supported */ +}; + + +/* the bus data */ +bus_datum bus_data[NBUS]; + +/* the IO INT data, one entry per possible APIC INTerrupt */ +io_int io_apic_ints[NINTR]; + +static int nintrs; + +#if defined(FIX_MP_TABLE_WORKS) +static void fix_mp_table __P((void)); +#endif /* FIX_MP_TABLE_WORKS */ + +static void processor_entry __P((proc_entry_ptr entry, int *cpu)); +static void io_apic_entry __P((io_apic_entry_ptr entry, int *apic)); +static void bus_entry __P((bus_entry_ptr entry, int *bus)); +static void int_entry __P((int_entry_ptr entry, int *intr)); +static int lookup_bus_type __P((char *name)); + + +/* + * parse an Intel MP specification table + */ +static int +parse_mp_table(void) +{ + int x; + mpfps_t fps; + mpcth_t cth; + int totalSize; + void *position; + int count; + int type; + int apic, bus, cpu, intr; + + /* clear physical APIC ID to logical CPU/IO table */ + for (x = 0; x < NAPICID; ++x) + ID_TO_IO(x) = -1; + + /* clear logical CPU to APIC ID table */ + for (x = 0; x < NCPU; ++x) + CPU_TO_ID(x) = -1; + + /* clear logical IO to APIC ID table */ + for (x = 0; x < NAPIC; ++x) + IO_TO_ID(x) = -1; + + /* clear IO APIC address table */ + for (x = 0; x < NAPIC; ++x) + io_apic_address[x] = ~0; + + /* clear bus data table */ + for (x = 0; x < NBUS; ++x) + bus_data[x].bus_id = 0xff; + + /* clear IO APIC INT table */ + for (x = 0; x < NINTR; ++x) + io_apic_ints[x].int_type = 0xff; + nintrs = 0; + + /* count the BSP */ + mp_ncpus = 1; + + /* setup the cpu/apic mapping arrays */ + boot_cpu_id = -1; + + /* local pointer */ + fps = (mpfps_t) mpfps; + + /* record whether PIC or virtual-wire mode */ + picmode = (fps->mpfb2 & 0x80) ? 1 : 0; + + /* check for use of 'default' configuration */ +#if defined(TEST_DEFAULT_CONFIG) + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return TEST_DEFAULT_CONFIG; +#else + if (fps->mpfb1 != 0) { + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return fps->mpfb1; + } +#endif /* TEST_DEFAULT_CONFIG */ + + if ((cth = fps->pap) == 0) + panic("MP Configuration Table Header MISSING!\n"); + + cpu_apic_address = (vm_offset_t) cth->apic_address; + + totalSize = cth->base_table_length - sizeof(struct MPCTH); + position = (u_char *) cth + sizeof(struct MPCTH); + count = cth->entry_count; + + apic = 0; /* logical apic# start @ 0 */ + bus = 0; /* logical bus# start @ 0 */ + cpu = 1; /* logical cpu# start @ 0, BUT reserve 0 for */ + /* BSP */ + intr = 0; /* unknown */ + + /* walk the table, recording info of interest */ + while (count--) { + switch (type = *(u_char *) position) { + case 0: + processor_entry(position, &cpu); + break; + case 1: + bus_entry(position, &bus); + break; + case 2: + io_apic_entry(position, &apic); + break; + case 3: + int_entry(position, &intr); + break; + case 4: + /* int_entry(position); */ + break; + default: + panic("mpfps Base Table HOSED!\n"); + /* NOTREACHED */ + } + + totalSize -= basetable_entry_types[type].length; + (u_char *) position += basetable_entry_types[type].length; + } + + if (boot_cpu_id == -1) + panic("NO BSP found!\n"); + + /* record # of APs found */ + mp_naps = (cpu - 1); + + /* record # of busses found */ + mp_nbusses = bus; + + /* record # of IO APICs found */ + mp_napics = apic; + + /* record # of IO APICs found */ + nintrs = intr; + +#if defined(FIX_MP_TABLE_WORKS) + /* post scan cleanup */ + fix_mp_table(); +#endif /* FIX_MP_TABLE_WORKS */ + + /* report fact that its NOT a default configuration */ + return 0; +} + + +/* + * parse an Intel MP specification table + */ +#if defined(FIX_MP_TABLE_WORKS) +static void +fix_mp_table(void) +{ + int x; + int y; + int num_pci_bus; + bus_datum bus_record; + + /* + * Fix mis-numbering of the PCI bus and its INT entries if the BIOS + * did it wrong. The MP spec says that when more than 1 PCI bus + * exists the BIOS must begin with bus entries for the PCI bus and use + * actual PCI bus numbering. This implies that when only 1 PCI bus + * exists the BIOS can choose to ignore this ordering, and indeed many + * MP motherboards do ignore it. This causes a problem when the PCI + * sub-system makes requests of the MP sub-system based on PCI bus + * numbers. So here we look for the situation and renumber the + * busses and associated INTs in an effort to "make it right". + */ + + /* count the number of PCI busses */ + for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + ++num_pci_bus; + } + + /* check the 1 PCI bus case for sanity */ + if (num_pci_bus == 1) { + + /* if its in the first slot all is well */ + if (bus_data[0].bus_type == PCI) + return; + + /* mis-numbered, swap with whichever bus uses slot 0 */ + + /* locate the entry holding the PCI bus */ + for (x = 1; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + break; + } + + /* swap the bus entry records */ + bus_record = bus_data[0]; + bus_data[0] = bus_data[x]; + bus_data[x] = bus_record; + + /* swap each relavant INTerrupt entry */ + for (y = 0; y < nintrs; ++y) { + if (io_apic_ints[y].src_bus_id == x) + io_apic_ints[y].src_bus_id = 0; + else + if (io_apic_ints[y].src_bus_id == 0) + io_apic_ints[y].src_bus_id = x; + } + } + /* sanity check if more than 1 PCI bus */ + else + if (num_pci_bus > 1) { + for (x = 0; x < num_pci_bus; ++x) { + if (bus_data[x].bus_type != PCI) { + printf("bad PCI bus numbering\n"); + panic("\n"); + } + } + } +} +#endif /* FIX_MP_TABLE_WORKS */ + + +static void +processor_entry(proc_entry_ptr entry, int *cpu) +{ + int x = *cpu; + + /* check for usability */ + if (!(entry->cpu_flags & PROCENTRY_FLAG_EN)) + return; + + /* check for BSP flag */ + if (entry->cpu_flags & PROCENTRY_FLAG_BP) { + /* always give boot CPU the logical value of 0 */ + x = 0; + boot_cpu_id = entry->apic_id; + } else { + /* add another AP to list, if less than max number of CPUs */ + if (x == NCPU) { + printf("Warning: only using %d of the available CPUs!\n", x); + return; + } + ++(*cpu); + } + + CPU_TO_ID(x) = entry->apic_id; + ID_TO_CPU(entry->apic_id) = x; +} + + +static void +bus_entry(bus_entry_ptr entry, int *bus) +{ + int x, y; + char name[8]; + char c; + + if ((x = (*bus)++) == NBUS) + panic("too many busses, increase 'NBUS'\n"); + + /* encode the name into an index */ + for (y = 0; y < 6; ++y) { + if ((c = entry->bus_type[y]) == ' ') + break; + name[y] = c; + } + name[y] = '\0'; + + if ((y = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) + panic("unknown bus type: '%s'\n", name); + + bus_data[x].bus_id = entry->bus_id; + bus_data[x].bus_type = y; +} + + +static void +io_apic_entry(io_apic_entry_ptr entry, int *apic) +{ + int x; + + if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) + return; + + if ((x = (*apic)++) == NAPIC) + panic("too many APICs, increase 'NAPIC'\n"); + + IO_TO_ID(x) = entry->apic_id; + ID_TO_IO(entry->apic_id) = x; + + io_apic_address[x] = (vm_offset_t) entry->apic_address; +} + + +static int +lookup_bus_type(char *name) +{ + int x; + + for (x = 0; x < MAX_BUSTYPE; ++x) + if (strcmp(bus_type_table[x].name, name) == 0) + return bus_type_table[x].type; + + return UNKNOWN_BUSTYPE; +} + + +static void +int_entry(int_entry_ptr entry, int *intr) +{ + int x; + + if ((x = (*intr)++) == NINTR) + panic("too many INTs, increase 'NINTR'\n"); + + io_apic_ints[x].int_type = entry->int_type; + io_apic_ints[x].int_flags = entry->int_flags; + io_apic_ints[x].src_bus_id = entry->src_bus_id; + io_apic_ints[x].src_bus_irq = entry->src_bus_irq; + io_apic_ints[x].dst_apic_id = entry->dst_apic_id; + io_apic_ints[x].dst_apic_int = entry->dst_apic_int; +} + + +static int +apic_int_is_bus_type(int intr, int bus_type) +{ + int bus; + + for (bus = 0; bus < mp_nbusses; ++bus) + if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) + && ((int) bus_data[bus].bus_type == bus_type)) + return 1; + + return 0; +} + + +/* + * determine which APIC pin an ISA INT is attached to. + */ +#define INTTYPE(I) (io_apic_ints[(I)].int_type) +#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) + +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_isa_apic_irq(int isaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (isaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == isaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, ISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin an EISA INT is attached to. + */ +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_eisa_apic_irq(int eisaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (eisaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == eisaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, EISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin a PCI INT is attached to. + */ +#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) +#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) +#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) +int +get_pci_apic_irq(int pciBus, int pciDevice, int pciInt) +{ + int intr; + + --pciInt; /* zero based */ + + for (intr = 0; intr < nintrs; ++intr) /* search each record */ + if ((INTTYPE(intr) == 0) +#if defined(FIX_MP_TABLE_WORKS) + && (SRCBUSID(intr) == pciBus) +#endif /* FIX_MP_TABLE_WORKS */ + && (SRCBUSDEVICE(intr) == pciDevice) + && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, PCI)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSLINE +#undef SRCBUSDEVICE +#undef SRCBUSID + +#undef INTPIN +#undef INTTYPE + + +int +undirect_pci_irq(int rirq) +{ +#if defined(READY) + printf("Freeing irq %d for ISA cards.\n", rirq); + /** FIXME: tickle the MB redirector chip */ + return ???; +#else + printf("Freeing (NOT implimented) irq %d for ISA cards.\n", rirq); + return 0; +#endif /* READY */ +} + + +/* + * given a bus ID, return: + * the bus type if found + * -1 if NOT found + */ +int +apic_bus_type(int id) +{ + int x; + + for (x = 0; x < mp_nbusses; ++x) + if (bus_data[x].bus_id == id) + return bus_data[x].bus_type; + + return -1; +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus ID if found + * -1 if NOT found + */ +int +apic_src_bus_id(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_id); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus IRQ if found + * -1 if NOT found + */ +int +apic_src_bus_irq(int apic, int pin) +{ + int x; + + for (x = 0; x < nintrs; x++) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_irq); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated INTerrupt type if found + * -1 if NOT found + */ +int +apic_int_type(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_type); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated trigger mode if found + * -1 if NOT found + */ +int +apic_trigger(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return ((io_apic_ints[x].int_flags >> 2) & 0x03); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated 'active' level if found + * -1 if NOT found + */ +int +apic_polarity(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_flags & 0x03); + + return -1; /* NOT found */ +} + + +/* + * set data according to MP defaults + * FIXME: probably not complete yet... + */ +static void +default_mp_table(int type) +{ + int ap_cpu_id; +#if defined(APIC_IO) + u_int32_t ux; + int io_apic_id; + int pin; +#endif /* APIC_IO */ + +#if 0 + printf(" MP default config type: %d\n", type); + switch (type) { + case 1: + printf(" bus: ISA, APIC: 82489DX\n"); + break; + case 2: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 3: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 4: + printf(" bus: MCA, APIC: 82489DX\n"); + break; + case 5: + printf(" bus: ISA+PCI, APIC: Integrated\n"); + break; + case 6: + printf(" bus: EISA+PCI, APIC: Integrated\n"); + break; + case 7: + printf(" bus: MCA+PCI, APIC: Integrated\n"); + break; + default: + printf(" future type\n"); + break; + /* NOTREACHED */ + } +#endif /* 0 */ + + boot_cpu_id = (apic_base[APIC_ID] & APIC_ID_MASK) >> 24; + ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; + + /* BSP */ + CPU_TO_ID(0) = boot_cpu_id; + ID_TO_CPU(boot_cpu_id) = 0; + + /* one and only AP */ + CPU_TO_ID(1) = ap_cpu_id; + ID_TO_CPU(ap_cpu_id) = 1; + mp_naps = 1; + + /* one and only IO APIC */ +#if defined(APIC_IO) + io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; + + /* + * sanity check, refer to MP spec section 3.6.6, last paragraph + * necessary as some hardware isn't properly setting up the IO APIC + */ +#if defined(REALLY_ANAL_IOAPICID_VALUE) + if (io_apic_id != 2) { +#else + if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { +#endif /* REALLY_ANAL_IOAPICID_VALUE */ + ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ + ux &= ~APIC_ID_MASK; /* clear the ID field */ + ux |= 0x02000000; /* set it to '2' */ + io_apic_write(0, IOAPIC_ID, ux); /* write new value */ + ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ + if ((ux & APIC_ID_MASK) != 0x02000000) + panic("Problem: can't control IO APIC ID, reg: 0x%08x\n", ux); + io_apic_id = 2; + } + IO_TO_ID(0) = io_apic_id; + ID_TO_IO(io_apic_id) = 0; + mp_napics = 1; +#else + mp_napics = 0; +#endif /* APIC_IO */ + + /* fill out bus entries */ + switch (type) { + case 1: + case 2: + case 3: + case 5: + case 6: + mp_nbusses = default_data[type - 1][0]; + bus_data[0].bus_id = default_data[type - 1][1]; + bus_data[0].bus_type = default_data[type - 1][2]; + bus_data[1].bus_id = default_data[type - 1][3]; + bus_data[1].bus_type = default_data[type - 1][4]; + break; + + /* case 4: case 7: MCA NOT supported */ + default: /* illegal/reserved */ + panic("BAD default MP config: %d\n", type); + } + +#if defined(APIC_IO) + /* general cases from MP v1.4, table 5-2 */ + for (pin = 0; pin < 16; ++pin) { + io_apic_ints[pin].int_type = 0; + io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ + io_apic_ints[pin].src_bus_id = 0; + io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ + io_apic_ints[pin].dst_apic_id = io_apic_id; + io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ + } + + /* special cases from MP v1.4, table 5-2 */ + if (type == 2) { + io_apic_ints[2].int_type = 0xff; /* N/C */ + io_apic_ints[13].int_type = 0xff; /* N/C */ +#if !defined(APIC_MIXED_MODE) + /** FIXME: ??? */ + panic("sorry, can't support type 2 default yet\n"); +#endif /* APIC_MIXED_MODE */ + } else + io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ + + if (type == 7) + io_apic_ints[0].int_type = 0xff; /* N/C */ + else + io_apic_ints[0].int_type = 3; /* vectored 8259 */ + + nintrs = 16; +#endif /* APIC_IO */ +} + + +static void install_ap_tramp(u_int boot_addr); +static int start_ap(int logicalCpu, u_int boot_addr); + +/* + * start each AP in our list + */ +static int +start_all_aps(u_int boot_addr) +{ + int x; + u_char mpbiosreason; + u_long mpbioswarmvec; + + /** + * NOTE: this needs further thought: + * where does it get released? + * should it be set to empy? + * + * get the initial mp_lock with a count of 1 for the BSP + */ + mp_lock = (apic_base[APIC_ID] & APIC_ID_MASK) + 1; + + /* initialize BSP's local APIC */ + apic_initialize(1); + + /* install the AP 1st level boot code */ + install_ap_tramp(boot_addr); + + /* save the current value of the warm-start vector */ + mpbioswarmvec = *((u_long *) WARMBOOT_OFF); + outb(CMOS_REG, BIOS_RESET); + mpbiosreason = inb(CMOS_DATA); + + /* start each AP */ + for (x = 1; x <= mp_naps; ++x) { + + /* setup a vector to our boot code */ + *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; + *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ + + /* attempt to start the Application Processor */ + CHECK_INIT(99); /* setup checkpoints */ + if (!start_ap(x, boot_addr)) { + printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); + CHECK_PRINT("trace"); /* show checkpoints */ + /* + * better panic as the AP may be running loose + * somewhere + */ + printf("panic y/n? [n] "); + if (cngetc() != 'n') + panic("bye-bye\n"); + } + CHECK_PRINT("trace"); /* show checkpoints */ + + /* record its version info */ + cpu_apic_versions[x] = cpu_apic_versions[0]; + } + + /* fill in our (BSP) APIC version */ + cpu_apic_versions[0] = apic_base[APIC_VER]; + + /* restore the warmstart vector */ + *(u_long *) WARMBOOT_OFF = mpbioswarmvec; + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, mpbiosreason); + + /* number of APs actually started */ + return mp_ncpus - 1; +} + + +/* + * load the 1st level AP boot code into base memory. + */ + +/* targets for relocation */ +extern void bigJump(void); +extern void bootCodeSeg(void); +extern void bootDataSeg(void); +extern void MPentry(void); +extern u_int MP_GDT; +extern u_int mp_gdtbase; + +static void +install_ap_tramp(u_int boot_addr) +{ + int x; + int size = *(int *) ((u_long) & bootMP_size); + u_char *src = (u_char *) ((u_long) bootMP); + u_char *dst = (u_char *) boot_addr + KERNBASE; + u_int boot_base = (u_int) bootMP; + u_int8_t *dst8; + u_int16_t *dst16; + u_int32_t *dst32; + + for (x = 0; x < size; ++x) + *dst++ = *src++; + + /* + * modify addresses in code we just moved to basemem. unfortunately we + * need fairly detailed info about mpboot.s for this to work. changes + * to mpboot.s might require changes here. + */ + + /* boot code is located in KERNEL space */ + dst = (u_char *) boot_addr + KERNBASE; + + /* modify the lgdt arg */ + dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); + *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); + + /* modify the ljmp target for MPentry() */ + dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); + *dst32 = ((u_int) MPentry - KERNBASE); + + /* modify the target for boot code segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; + + /* modify the target for boot data segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; +} + + +/* + * this function starts the AP (application processor) identified + * by the APIC ID 'physicalCpu'. It does quite a "song and dance" + * to accomplish this. This is necessary because of the nuances + * of the different hardware we might encounter. It ain't pretty, + * but it seems to work. + */ +static int +start_ap(int logical_cpu, u_int boot_addr) +{ + int physical_cpu; + int vector; + int cpus; + u_long icr_lo, icr_hi; + + /* get the PHYSICAL APIC ID# */ + physical_cpu = CPU_TO_ID(logical_cpu); + + /* calculate the vector */ + vector = (boot_addr >> 12) & 0xff; + + /* used as a watchpoint to signal AP startup */ + cpus = mp_ncpus; + + /* + * first we do an INIT/RESET IPI this INIT IPI might be run, reseting + * and running the target CPU. OR this INIT IPI might be latched (P5 + * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be + * ignored. + */ + + /* setup the address for the target AP */ + icr_hi = apic_base[APIC_ICR_HI] & ~APIC_ID_MASK; + icr_hi |= (physical_cpu << 24); + apic_base[APIC_ICR_HI] = icr_hi; + + /* do an INIT IPI: assert RESET */ + icr_lo = apic_base[APIC_ICR_LOW] & 0xfff00000; + apic_base[APIC_ICR_LOW] = icr_lo | 0x0000c500; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* do an INIT IPI: deassert RESET */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00008500; + + /* wait for pending status end */ + u_sleep(10000); /* wait ~10mS */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* + * next we do a STARTUP IPI: the previous INIT IPI might still be + * latched, (P5 bug) this 1st STARTUP would then terminate + * immediately, and the previously started INIT IPI would continue. OR + * the previous INIT IPI has already run. and this STARTUP IPI will + * run. OR the previous INIT IPI was ignored. and this STARTUP IPI + * will run. + */ + + /* do a STARTUP IPI */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* + * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF + * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR + * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is + * recognized after hardware RESET or INIT IPI. + */ + + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* wait for it to start */ + set_apic_timer(5000000);/* == 5 seconds */ + while (read_apic_timer()) + if (mp_ncpus > cpus) + return 1; /* return SUCCESS */ + + return 0; /* return FAILURE */ +} + + +#if defined(IPI_INTS) + +static void +ipi_intr0(void) +{ + printf("IPI 0\n"); +} + + +static void +ipi_intr1(void) +{ + printf("IPI 1\n"); +} + + +static void +ipi_intr2(void) +{ + printf("IPI 2\n"); +} + + +static void +ipi_intr3(void) +{ + printf("IPI 3\n"); +} + +/*----------------------------------------------------------------------- +** +** Register an interupt handler for an IPI. +** (Stolen from the PCI<->ISA glue code) +** +**----------------------------------------------------------------------- +*/ + +static int +ipi_ihandler_attach(int irq, inthand2_t * func, unsigned *maskptr, int unit) +{ + char buf[16]; + char *cp; + int free_id, id, result; + + sprintf(buf, "ipi irq%d", irq); + for (cp = intrnames, free_id = 0, id = 0; id < NR_DEVICES; id++) { + if (strcmp(cp, buf) == 0) + break; + if (free_id <= 0 && strcmp(cp, "ipi irqnn") == 0) + free_id = id; + while (*cp++ != '\0'); + } + if (id == NR_DEVICES) { + id = free_id; + if (id == 0) { + /* + * All ipi irq counters are in use, perhaps because + * config is old so there aren't any. Abuse the clk0 + * counter. + */ + printf( + "ipi_ihandler_attach: counting ipi irq%d's as clk0 irqs\n", + irq); + } + } + result = register_intr( + irq, /* isa irq */ + id, /* device id */ + 0, /* flags? */ + func, /* handler */ + maskptr, /* mask pointer */ + unit); /* handler arg */ + + if (result) { + printf("WARNING: ipi_ihandler_attach: result=%d\n", result); + return (result); + }; + + return (0); +} +#endif /* IPI_INTS */ + + +#ifdef SMP_INVLTLB +/* + * Flush the TLB on all other CPU's + * + * XXX: Needs to handshake and wait for completion before proceding. + */ + +void +smp_invltlb() +{ + if (smp_active) { + if (invldebug & 2) +#if defined(XFAST_IPI32) + all_but_self_ipi(ICU_OFFSET + 32); +#else + all_but_self_ipi(ICU_OFFSET + 27); +#endif /* XFAST_IPI32 */ + } +} + +void +invlpg(u_int addr) +{ + __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); + smp_invltlb(); +} + +void +invltlb(void) +{ + u_long temp; + /* + * This should be implemented as load_cr3(rcr3()) when load_cr3() is + * inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); + smp_invltlb(); +} +/* + * Handles recieving an "IRQ 27", the invalidate tlb IPI.. + */ +void +ipi_invltlb(void) +{ + u_long temp; + + if (invldebug & 4) { + /* + * This should be implemented as load_cr3(rcr3()) when + * load_cr3() is inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) + :: "memory"); + + } +} +#endif /* SMP_INVLTLB */ diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h index ecfd8b2c315d..73ee223404cc 100644 --- a/sys/amd64/include/pcb.h +++ b/sys/amd64/include/pcb.h @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)pcb.h 5.10 (Berkeley) 5/12/91 - * $Id: pcb.h,v 1.18 1997/02/22 09:34:56 peter Exp $ + * $Id: pcb.h,v 1.19 1997/04/07 06:45:18 peter Exp $ */ #ifndef _I386_PCB_H_ @@ -60,6 +60,8 @@ struct pcb { u_char pcb_flags; #define FP_SOFTFP 0x01 /* process using software fltng pnt emulator */ caddr_t pcb_onfault; /* copyin/out fault recovery */ + u_long pcb_mpnest; + u_long __pcb_spare[7]; /* adjust to avoid core dump size changes */ #if 0 /* some day we may switch between procs that have their own i386tss */ struct i386tss pcb_tss; u_char pcb_iomap[NPORT/sizeof(u_char)]; /* i/o port bitmap */ @@ -74,7 +76,15 @@ struct md_coredump { }; #ifdef KERNEL + +#include "opt_smp.h" +#include <machine/smp.h> +#ifdef SMP +#define curpcb (SMPcurpcb[cpunumber()]) +#else /* !SMP */ extern struct pcb *curpcb; /* our current running pcb */ +#endif /* SMP */ + void savectx __P((struct pcb*)); #endif diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h index 6b27cd28ac34..f0d4c017178b 100644 --- a/sys/amd64/include/pmap.h +++ b/sys/amd64/include/pmap.h @@ -42,7 +42,7 @@ * * from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90 * from: @(#)pmap.h 7.4 (Berkeley) 5/12/91 - * $Id: pmap.h,v 1.48 1997/02/22 09:34:58 peter Exp $ + * $Id: pmap.h,v 1.49 1997/04/07 09:30:20 peter Exp $ */ #ifndef _MACHINE_PMAP_H_ @@ -91,7 +91,11 @@ #define NKPT 9 /* actual number of kernel page tables */ #endif #ifndef NKPDE +#if defined(SMP) && defined(SMP_PRIVPAGES) +#define NKPDE 62 /* addressable number of page tables/pde's */ +#else /* SMP && SMP_PRIVPAGES */ #define NKPDE 63 /* addressable number of page tables/pde's */ +#endif /* SMP && SMP_PRIVPAGES */ #endif /* @@ -99,9 +103,16 @@ * * XXX This works for now, but I am not real happy with it, I'll fix it * right after I fix locore.s and the magic 28K hole + * + * SMP_PRIVPAGES: The per-cpu address space is 0xff80000 -> 0xffbfffff */ #define APTDPTDI (NPDEPG-1) /* alt ptd entry that points to APTD */ +#if defined(SMP) && defined(SMP_PRIVPAGES) +#define MPPTDI (APTDPTDI-1) /* per cpu ptd entry */ +#define KPTDI (MPPTDI-NKPDE) /* start of kernel virtual pde's */ +#else /* SMP && SMP_PRIVPAGES */ #define KPTDI (APTDPTDI-NKPDE)/* start of kernel virtual pde's */ +#endif /* SMP && SMP_PRIVPAGES */ #define PTDPTDI (KPTDI-1) /* ptd entry that points to ptd! */ #define UMAXPTDI (PTDPTDI-1) /* ptd entry for user space end */ #define UMAXPTEOFF (NPTEPG-UPAGES_HOLE) /* pte entry for user space end */ diff --git a/sys/amd64/include/segments.h b/sys/amd64/include/segments.h index 27525889053d..e1e6b4684c91 100644 --- a/sys/amd64/include/segments.h +++ b/sys/amd64/include/segments.h @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)segments.h 7.1 (Berkeley) 5/9/91 - * $Id$ + * $Id: segments.h,v 1.13 1997/02/22 09:35:09 peter Exp $ */ #ifndef _MACHINE_SEGMENTS_H_ @@ -234,7 +234,7 @@ struct region_descriptor { #ifdef KERNEL extern int currentldt; extern int _default_ldt; -extern union descriptor gdt[NGDT]; +extern union descriptor gdt[]; extern struct soft_segment_descriptor gdt_segs[]; extern struct gate_descriptor idt[NIDT]; extern union descriptor ldt[NLDT]; diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h new file mode 100644 index 000000000000..905ca3529327 --- /dev/null +++ b/sys/amd64/include/smp.h @@ -0,0 +1,135 @@ +/* + * ---------------------------------------------------------------------------- + * "THE BEER-WARE LICENSE" (Revision 42): + * <phk@FreeBSD.org> wrote this file. As long as you retain this notice you + * can do whatever you want with this stuff. If we meet some day, and you think + * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp + * ---------------------------------------------------------------------------- + * + * $Id: smp.h,v 1.29 1997/04/26 08:11:50 peter Exp $ + * + */ + +#ifndef _MACHINE_SMP_H_ +#define _MACHINE_SMP_H_ + +#ifdef KERNEL + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" /* aiee! (for cpufunc.h!) */ + +#if defined(SMP) && !defined(NCPU) +# define NCPU 2 +#endif /* SMP && NCPU */ + +#if defined(SMP) || defined(APIC_IO) + +/* global data in mpboot.s */ +extern int bootMP_size; + +/* functions in mpboot.s */ +void bootMP __P((void)); + +/* global data in mplock.s */ +extern u_int mp_lock; + +/* functions in mplock.s */ +void get_mplock __P((void)); +void rel_mplock __P((void)); +void expect_mplock __P((void)); + +/* global data in mp_machdep.c */ +extern struct proc* SMPcurproc[NCPU]; +extern struct pcb* SMPcurpcb[NCPU]; +extern struct timeval SMPruntime[NCPU]; +extern int mp_ncpus; +extern int mp_naps; +extern int mp_nbusses; +extern int mp_napics; +extern int mp_picmode; +extern int mpenabled; +extern int boot_cpu_id; +extern vm_offset_t cpu_apic_address; +extern vm_offset_t io_apic_address[]; +extern u_int32_t cpu_apic_versions[]; +extern u_int32_t io_apic_versions[]; +extern int cpu_num_to_apic_id[]; +extern int io_num_to_apic_id[]; +extern int apic_id_to_logical[]; + +/* functions in mp_machdep.c */ +u_int mp_bootaddress __P((u_int)); +void mp_start __P((void)); +void mp_announce __P((void)); +int get_isa_apic_irq __P((int)); +int get_eisa_apic_irq __P((int)); +int get_pci_apic_irq __P((int, int, int)); +int undirect_pci_irq __P((int)); +int apic_bus_type __P((int)); +int apic_src_bus_id __P((int, int)); +int apic_src_bus_irq __P((int, int)); +int apic_int_type __P((int, int)); +int apic_trigger __P((int, int)); +int apic_polarity __P((int, int)); +void configure_local_apic __P((void)); +void init_secondary __P((void)); +#ifdef SMP_INVLTLB +void ipi_invltlb __P((void)); +void smp_invltlb __P((void)); +#endif + +/* global data in mpapic.c */ +extern volatile u_int* apic_base; + +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX +#else +extern volatile u_int* io_apic_base; +#endif /* MULTIPLE_IOAPICS */ + +/* functions in mpapic.c */ +void apic_initialize __P((int)); +int selected_apic_ipi __P((u_int, int, int)); +int io_apic_setup __P((int)); +int ext_int_setup __P((int, int)); +void write_io_apic_mask24 __P((int, u_int32_t)); + +#if defined(READY) +void clr_io_apic_mask24 __P((int, u_int32_t)); +void set_io_apic_mask24 __P((int, u_int32_t)); +#endif /* READY */ + +void set_apic_timer __P((int)); +int read_apic_timer __P((void)); +void u_sleep __P((int)); + +/* global data in init_smp.c */ +extern int smp_active; +extern int invldebug; + +/* in pmap.c FIXME: belongs in pmap.h??? */ +void pmap_bootstrap_apics __P((void)); +void pmap_bootstrap2 __P((void)); + +#if 0 +/* chicken and egg problem... */ +static __inline unsigned +cpunumber(void) +{ + return (unsigned)ID_TO_CPU((apic_base[APIC_ID] & APIC_ID_MASK) >> 24); +} +#else +/* + * we 'borrow' this info from apic.h + * this will go away soon... + */ +static __inline unsigned +cpunumber(void) +{ + return (unsigned)(apic_id_to_logical[(apic_base[8] & 0x0f000000) >> 24]); +} +#endif /* 0 */ + +#endif /* SMP || APIC_IO */ +#endif /* KERNEL */ +#endif /* _MACHINE_SMP_H_ */ diff --git a/sys/amd64/isa/clock.c b/sys/amd64/isa/clock.c index b9425536d67a..d46f982806bd 100644 --- a/sys/amd64/isa/clock.c +++ b/sys/amd64/isa/clock.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)clock.c 7.2 (Berkeley) 5/12/91 - * $Id: clock.c,v 1.79 1997/03/05 08:08:48 bde Exp $ + * $Id: clock.c,v 1.80 1997/04/06 13:25:48 mckay Exp $ */ /* @@ -49,6 +49,7 @@ */ #include "opt_clock.h" +#include "opt_smp.h" #include "opt_cpu.h" #include <sys/param.h> @@ -100,7 +101,7 @@ int adjkerntz; /* local offset from GMT in seconds */ int disable_rtc_set; /* disable resettodr() if != 0 */ u_int idelayed; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) u_int i586_ctr_bias; u_int i586_ctr_comultiplier; u_int i586_ctr_freq; @@ -143,7 +144,7 @@ static u_char timer0_state; static u_char timer2_state; static void (*timer_func) __P((struct clockframe *frame)) = hardclock; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) static void set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq); #endif static void set_timer_freq(u_int freq, int intr_freq); @@ -553,7 +554,7 @@ calibrate_clocks(void) goto fail; tot_count = 0; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) if (cpu_class == CPUCLASS_586 || cpu_class == CPUCLASS_686) wrmsr(0x10, 0LL); /* XXX 0x10 is the MSR for the TSC */ #endif @@ -586,7 +587,7 @@ calibrate_clocks(void) goto fail; } -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * Read the cpu cycle counter. The timing considerations are * similar to those for the i8254 clock. @@ -667,14 +668,14 @@ startrtclock() printf( "%d Hz differs from default of %d Hz by more than 1%%\n", freq, timer_freq); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) i586_ctr_freq = 0; #endif } set_timer_freq(timer_freq, hz); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) #ifndef CLK_USE_I586_CALIBRATION if (i586_ctr_freq != 0) { if (bootverbose) @@ -821,6 +822,20 @@ resettodr() writertc(RTC_STATUSB, rtc_statusb); } +#if defined(APIC_IO) + +/* from icu.s: */ +extern u_int hwisrs[]; +extern void vec8254 __P((void)); +extern void vecRTC __P((void)); +extern u_int ivectors[]; +extern u_int Xintr8254; +extern u_int XintrRTC; +extern u_int mask8254; +extern u_int maskRTC; + +#endif /* APIC_IO */ + /* * Start both clocks running. */ @@ -828,6 +843,9 @@ void cpu_initclocks() { int diag; +#if defined(APIC_IO) + int x; +#endif /* APIC_IO */ if (statclock_disable) { /* @@ -845,11 +863,47 @@ cpu_initclocks() } /* Finish initializing 8253 timer 0. */ +#if defined(APIC_IO) + /* 8254 is traditionally on ISA IRQ0 */ + if ((x = get_isa_apic_irq(0)) < 0) { + /* + * bummer, this mb doesn't have the 8254 on ISA irq0, + * perhaps it's on the EISA bus... + */ + if ((x = get_eisa_apic_irq(0)) < 0) { + /* double bummer, attempt to redirect thru the 8259 */ + if (bootverbose) + printf("APIC missing 8254 connection\n"); + + /* allow 8254 timer to INTerrupt 8259 */ +#if !defined(IO_ICU1) +#define IO_ICU1 0x20 +#endif + x = inb(IO_ICU1 + 1); /* current mask in 8259 */ + x &= ~1; /* clear 8254 timer mask */ + outb(IO_ICU1 + 1, x); /* write new mask */ + + /* program IO APIC for type 3 INT on INT0 */ + if (ext_int_setup(0, 0) < 0) + panic("8254 redirect impossible!"); + x = 0; /* 8259 is on 0 */ + } + } + + hwisrs[x] = (u_int)vec8254; + Xintr8254 = (u_int)ivectors[x]; /* XXX might need Xfastintr# */ + mask8254 = (1 << x); + register_intr(/* irq */ x, /* XXX id */ 0, /* flags */ 0, + /* XXX */ (inthand2_t *)clkintr, &clk_imask, + /* unit */ 0); + INTREN(mask8254); +#else register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0, /* XXX */ (inthand2_t *)clkintr, &clk_imask, /* unit */ 0); INTREN(IRQ0); -#if defined(I586_CPU) || defined(I686_CPU) +#endif /* APIC_IO */ +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * Finish setting up anti-jitter measures. */ @@ -867,11 +921,36 @@ cpu_initclocks() diag = rtcin(RTC_DIAG); if (diag != 0) printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS); +#if defined(APIC_IO) + /* RTC is traditionally on ISA IRQ8 */ + if ((x = get_isa_apic_irq(8)) < 0) { + if ((x = get_eisa_apic_irq(8)) < 0) { + panic("APIC missing RTC connection"); + } + } + + hwisrs[x] = (u_int)vecRTC; + XintrRTC = (u_int)ivectors[x]; /* XXX might need Xfastintr# */ + maskRTC = (1 << x); + register_intr(/* irq */ x, /* XXX id */ 1, /* flags */ 0, + /* XXX */ (inthand2_t *)rtcintr, &stat_imask, + /* unit */ 0); + INTREN(maskRTC); +#else register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0, /* XXX */ (inthand2_t *)rtcintr, &stat_imask, /* unit */ 0); INTREN(IRQ8); +#endif /* APIC_IO */ writertc(RTC_STATUSB, rtc_statusb); + +#if defined(APIC_IO) + printf("Enabled INTs: "); + for (x = 0; x < 24; ++x) + if ((imen & (1 << x)) == 0) + printf("%d, ", x); + printf("imen: 0x%08x\n", imen); +#endif /* APIC_IO */ } void @@ -900,7 +979,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS if (timer0_state != 0) return (EBUSY); /* too much trouble to handle */ set_timer_freq(freq, hz); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) set_i586_ctr_freq(i586_ctr_freq, timer_freq); #endif } @@ -910,7 +989,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS SYSCTL_PROC(_machdep, OID_AUTO, i8254_freq, CTLTYPE_INT | CTLFLAG_RW, 0, sizeof(u_int), sysctl_machdep_i8254_freq, "I", ""); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) static void set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq) { @@ -949,4 +1028,4 @@ sysctl_machdep_i586_freq SYSCTL_HANDLER_ARGS SYSCTL_PROC(_machdep, OID_AUTO, i586_freq, CTLTYPE_INT | CTLFLAG_RW, 0, sizeof(u_int), sysctl_machdep_i586_freq, "I", ""); -#endif /* defined(I586_CPU) || defined(I686_CPU) */ +#endif /* (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) */ diff --git a/sys/amd64/isa/icu.h b/sys/amd64/isa/icu.h index ba721727f340..d0f148289dc3 100644 --- a/sys/amd64/isa/icu.h +++ b/sys/amd64/isa/icu.h @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)icu.h 5.6 (Berkeley) 5/9/91 - * $Id$ + * $Id: icu.h,v 1.10 1997/02/22 09:36:13 peter Exp $ */ /* @@ -47,15 +47,62 @@ #ifndef LOCORE +#include "opt_smp.h" /* so we dont have to change EVERY file including icu.h */ + /* * Interrupt "level" mechanism variables, masks, and macros */ extern unsigned imen; /* interrupt mask enable */ +#if defined(APIC_IO) + +# if !defined(_MACHINE_SMP_H_) +/** XXX what a hack, its this or include <machine/smp.h>! */ +void write_io_apic_mask24 __P((int, u_int32_t)); /* i386/i386/mpapic.c */ +# endif /* _MACHINE_SMP_H_ */ + +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX: cannot assume apic #0 in the following functions. +#endif /* MULTIPLE_IOAPICS */ + +static __inline u_int32_t +INTRGET( void ) +{ + return (imen & 0x00ffffff); /* return our global copy */ +} + +static __inline void +INTRSET( unsigned s ) +{ + write_io_apic_mask24( 0, s ); + imen = s; +} + +static __inline void +INTREN( unsigned s ) +{ + write_io_apic_mask24( 0, imen & ~s ); + imen &= ~s; +} + +static __inline void +INTRDIS( unsigned s ) +{ + write_io_apic_mask24( 0, imen | s ); + imen |= s; +} + + +#define INTRMASK(msk,s) (msk |= (s)) +#define INTRUNMASK(msk,s) (msk &= ~(s)) + +#else /* APIC_IO */ + #define INTREN(s) (imen &= ~(s), SET_ICUS()) #define INTRDIS(s) (imen |= (s), SET_ICUS()) #define INTRMASK(msk,s) (msk |= (s)) #define INTRUNMASK(msk,s) (msk &= ~(s)) + #if 0 #ifdef PC98 #define SET_ICUS() (outb(IO_ICU1 + 2, imen), outb(IU_ICU2 + 2, imen >> 8)) @@ -74,6 +121,8 @@ extern unsigned imen; /* interrupt mask enable */ #endif #endif +#endif /* APIC_IO */ + #endif /* LOCORE */ /* @@ -108,6 +157,22 @@ extern unsigned imen; /* interrupt mask enable */ * Interrupt Control offset into Interrupt descriptor table (IDT) */ #define ICU_OFFSET 32 /* 0-31 are processor exceptions */ + +#if defined(APIC_IO) + +#include <machine/apic.h> +#if defined(IPI_INTS) +/* 32-47: ISA IRQ0-IRQ15, 48-55: IO APIC IRQ16-IRQ23, 56-59: LOCAL APIC IPI */ +#define ICU_LEN 28 +#else +/* 32-47: ISA IRQ0-IRQ15, 48-55: IO APIC IRQ16-IRQ23 */ +#define ICU_LEN 24 +#endif /* IPI_INTS */ + +#else + #define ICU_LEN 16 /* 32-47 are ISA interrupts */ +#endif /* APIC_IO */ + #endif /* !_I386_ISA_ICU_H_ */ diff --git a/sys/amd64/isa/isa.c b/sys/amd64/isa/isa.c index b4fa75b6665a..8b809e257c8a 100644 --- a/sys/amd64/isa/isa.c +++ b/sys/amd64/isa/isa.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)isa.c 7.2 (Berkeley) 5/13/91 - * $Id: isa.c,v 1.79 1997/03/25 03:29:40 ache Exp $ + * $Id: isa.c,v 1.80 1997/03/28 01:02:17 ache Exp $ */ /* @@ -48,6 +48,7 @@ */ #include "opt_auto_eoi.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -56,6 +57,10 @@ #include <sys/malloc.h> #include <machine/md_var.h> #include <machine/segments.h> +#if defined(APIC_IO) +#include <machine/smp.h> +#include <machine/apic.h> +#endif /* APIC_IO */ #include <vm/vm.h> #include <vm/vm_param.h> #include <vm/pmap.h> @@ -65,6 +70,14 @@ #include <i386/isa/ic/i8237.h> #include "vector.h" +#ifdef APIC_IO +/* + * This is to accommodate "mixed-mode" programming for + * motherboards that don't connect the 8254 to the IO APIC. + */ +#define AUTO_EOI_1 +#endif + /* ** Register definitions for DMA controller 1 (channels 0..3): */ @@ -96,6 +109,16 @@ static inthand_t *fastintr[ICU_LEN] = { &IDTVEC(fastintr10), &IDTVEC(fastintr11), &IDTVEC(fastintr12), &IDTVEC(fastintr13), &IDTVEC(fastintr14), &IDTVEC(fastintr15) +#if defined(APIC_IO) + , &IDTVEC(fastintr16), &IDTVEC(fastintr17), + &IDTVEC(fastintr18), &IDTVEC(fastintr19), + &IDTVEC(fastintr20), &IDTVEC(fastintr21), + &IDTVEC(fastintr22), &IDTVEC(fastintr23) +#if defined(IPI_INTS) +/* XXX probably NOT needed, we register_intr(slowintr[I]) */ + , &IDTVEC(ipi24), &IDTVEC(ipi25), &IDTVEC(ipi26), &IDTVEC(ipi27) +#endif /* IPI_INTS */ +#endif /* APIC_IO */ }; static inthand_t *slowintr[ICU_LEN] = { @@ -103,6 +126,13 @@ static inthand_t *slowintr[ICU_LEN] = { &IDTVEC(intr4), &IDTVEC(intr5), &IDTVEC(intr6), &IDTVEC(intr7), &IDTVEC(intr8), &IDTVEC(intr9), &IDTVEC(intr10), &IDTVEC(intr11), &IDTVEC(intr12), &IDTVEC(intr13), &IDTVEC(intr14), &IDTVEC(intr15) +#if defined(APIC_IO) + , &IDTVEC(intr16), &IDTVEC(intr17), &IDTVEC(intr18), &IDTVEC(intr19), + &IDTVEC(intr20), &IDTVEC(intr21), &IDTVEC(intr22), &IDTVEC(intr23) +#if defined(IPI_INTS) + , &IDTVEC(ipi24), &IDTVEC(ipi25), &IDTVEC(ipi26), &IDTVEC(ipi27) +#endif /* IPI_INTS */ +#endif /* APIC_IO */ }; static void config_isadev __P((struct isa_device *isdp, u_int *mp)); @@ -920,18 +950,52 @@ struct isa_device *find_isadev(table, driverp, unit) /* * Return nonzero if a (masked) irq is pending for a given device. */ +#if defined(APIC_IO) + int isa_irq_pending(dvp) struct isa_device *dvp; { + /* read APIC IRR containing the 16 ISA INTerrupts */ +#if defined(TEST_UPPERPRIO) + if ((u_int32_t)dvp->id_irq == APIC_IRQ10) + return (int)(apic_base[APIC_IRR2] & 1); + else +#endif /** TEST_UPPERPRIO */ + return ((apic_base[APIC_IRR1] & 0x00ffffff) + & (u_int32_t)dvp->id_irq) ? 1 : 0; +} + +/* + * an 8259 specific routine, + * for use by boot probes in certain device drivers. + */ +int +icu_irq_pending(dvp) + struct isa_device *dvp; +{ unsigned id_irq; + id_irq = dvp->id_irq; + if (id_irq & 0xff) + return (inb(IO_ICU1) & id_irq); + return (inb(IO_ICU2) & (id_irq >> 8)); +} + +#else /* APIC_IO */ +int +isa_irq_pending(dvp) + struct isa_device *dvp; +{ + unsigned id_irq; id_irq = dvp->id_irq; if (id_irq & 0xff) return (inb(IO_ICU1) & id_irq); return (inb(IO_ICU2) & (id_irq >> 8)); } +#endif /* APIC_IO */ + int update_intr_masks(void) { @@ -939,7 +1003,11 @@ update_intr_masks(void) u_int mask,*maskptr; for (intr=0; intr < ICU_LEN; intr ++) { - if (intr==2) continue; +#if defined(APIC_IO) + /* no 8259 SLAVE to ignore */ +#else + if (intr==2) continue; /* ignore 8259 SLAVE output */ +#endif /* APIC_IO */ maskptr = intr_mptr[intr]; if (!maskptr) continue; *maskptr |= 1 << intr; @@ -971,7 +1039,11 @@ register_intr(intr, device_id, flags, handler, maskptr, unit) int id; u_int mask = (maskptr ? *maskptr : 0); +#if defined(APIC_IO) + if ((u_int)intr >= ICU_LEN /* no 8259 SLAVE to ignore */ +#else if ((u_int)intr >= ICU_LEN || intr == 2 +#endif /* APIC_IO */ || (u_int)device_id >= NR_DEVICES) return (EINVAL); if (intr_handler[intr] != isa_strayintr) @@ -983,9 +1055,24 @@ register_intr(intr, device_id, flags, handler, maskptr, unit) intr_mptr[intr] = maskptr; intr_mask[intr] = mask | (1 << intr); intr_unit[intr] = unit; +#if defined(TEST_UPPERPRIO) + if (intr == 10) { + printf("--- setting IRQ10 to IDT64\n"); + setidt(64, + flags & RI_FAST ? fastintr[intr] : slowintr[intr], + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + } + else { + printf("setting IRQ%02d to IDT%02d\n", intr, ICU_OFFSET+intr); + setidt(ICU_OFFSET + intr, + flags & RI_FAST ? fastintr[intr] : slowintr[intr], + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + } +#else setidt(ICU_OFFSET + intr, flags & RI_FAST ? fastintr[intr] : slowintr[intr], SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /** TEST_UPPERPRIO */ write_eflags(ef); for (cp = intrnames, id = 0; id <= device_id; id++) while (*cp++ != '\0') @@ -995,9 +1082,12 @@ register_intr(intr, device_id, flags, handler, maskptr, unit) if (intr < 10) { cp[-3] = intr + '0'; cp[-2] = ' '; - } else { + } else if (intr < 20) { cp[-3] = '1'; cp[-2] = intr - 10 + '0'; + } else { + cp[-3] = '2'; + cp[-2] = intr - 20 + '0'; } return (0); } diff --git a/sys/amd64/isa/npx.c b/sys/amd64/isa/npx.c index 6e3f5c77e077..402b1cebb76f 100644 --- a/sys/amd64/isa/npx.c +++ b/sys/amd64/isa/npx.c @@ -32,7 +32,7 @@ * SUCH DAMAGE. * * from: @(#)npx.c 7.2 (Berkeley) 5/12/91 - * $Id: npx.c,v 1.40 1997/03/24 11:23:58 bde Exp $ + * $Id: npx.c,v 1.41 1997/04/22 06:55:38 jdp Exp $ */ #include "npx.h" @@ -40,6 +40,7 @@ #include "opt_cpu.h" #include "opt_math_emulate.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -60,6 +61,10 @@ #include <machine/trap.h> #include <machine/clock.h> #include <machine/specialreg.h> +#if defined(APIC_IO) +#include <machine/apic.h> +#include <machine/mpapic.h> +#endif /* APIC_IO */ #include <i386/isa/icu.h> #include <i386/isa/isa_device.h> @@ -133,7 +138,12 @@ SYSCTL_INT(_hw,HW_FLOATINGPT, floatingpoint, "Floatingpoint instructions executed in hardware"); static u_int npx0_imask = SWI_CLOCK_MASK; +#ifdef SMP +#define npxproc (SMPnpxproc[cpunumber()]) +struct proc *SMPnpxproc[NCPU]; +#else struct proc *npxproc; +#endif static bool_t npx_ex16; static bool_t npx_exists; @@ -148,8 +158,28 @@ static volatile u_int npx_traps_while_probing; * interrupts. We'll still need a special exception 16 handler. The busy * latch stuff in probeintr() can be moved to npxprobe(). */ - inthand_t probeintr; + +#if defined(APIC_IO) + +asm +(" + .text + .p2align 2,0x90 +" __XSTRING(CNAME(probeintr)) ": + ss + incl " __XSTRING(CNAME(npx_intrs_while_probing)) " + pushl %eax + movl " __XSTRING(CNAME(apic_base)) ",%eax # EOI to local APIC + movl $0,0xb0(,%eax,1) # movl $0, APIC_EOI(%eax) + movb $0,%al + outb %al,$0xf0 # clear BUSY# latch + popl %eax + iret +"); + +#else + asm (" .text @@ -167,6 +197,8 @@ asm iret "); +#endif /* APIC_IO */ + inthand_t probetrap; asm (" @@ -191,8 +223,12 @@ npxprobe(dvp) { int result; u_long save_eflags; +#if defined(APIC_IO) + u_int save_apic_mask; +#else u_char save_icu1_mask; u_char save_icu2_mask; +#endif /* APIC_IO */ struct gate_descriptor save_idt_npxintr; struct gate_descriptor save_idt_npxtrap; /* @@ -205,20 +241,32 @@ npxprobe(dvp) npx_intrno = NRSVIDT + ffs(dvp->id_irq) - 1; save_eflags = read_eflags(); disable_intr(); +#if defined(APIC_IO) + save_apic_mask = INTRGET(); +#else save_icu1_mask = inb(IO_ICU1 + 1); save_icu2_mask = inb(IO_ICU2 + 1); +#endif /* APIC_IO */ save_idt_npxintr = idt[npx_intrno]; save_idt_npxtrap = idt[16]; +#if defined(APIC_IO) + INTRSET( ~dvp->id_irq ); +#else outb(IO_ICU1 + 1, ~(IRQ_SLAVE | dvp->id_irq)); outb(IO_ICU2 + 1, ~(dvp->id_irq >> 8)); +#endif /* APIC_IO */ setidt(16, probetrap, SDT_SYS386TGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); setidt(npx_intrno, probeintr, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); npx_idt_probeintr = idt[npx_intrno]; enable_intr(); result = npxprobe1(dvp); disable_intr(); +#if defined(APIC_IO) + INTRSET( save_apic_mask ); +#else outb(IO_ICU1 + 1, save_icu1_mask); outb(IO_ICU2 + 1, save_icu2_mask); +#endif /* APIC_IO */ idt[npx_intrno] = save_idt_npxintr; idt[16] = save_idt_npxtrap; write_eflags(save_eflags); @@ -364,7 +412,8 @@ npxattach(dvp) } npxinit(__INITIAL_NPXCW__); -#ifdef I586_CPU +#if defined(I586_CPU) && !defined(SMP) + /* FPU not working under SMP yet */ if (cpu_class == CPUCLASS_586 && npx_ex16) { if (!(dvp->id_flags & NPX_DISABLE_I586_OPTIMIZED_BCOPY)) { bcopy_vector = i586_bcopy; @@ -576,18 +625,32 @@ void npxsave(addr) struct save87 *addr; { +#if defined(APIC_IO) + u_int apic_mask; + u_int old_apic_mask; +#else u_char icu1_mask; u_char icu2_mask; u_char old_icu1_mask; u_char old_icu2_mask; +#endif /* APIC_IO */ struct gate_descriptor save_idt_npxintr; disable_intr(); +#if defined(APIC_IO) + old_apic_mask = INTRGET(); +#else old_icu1_mask = inb(IO_ICU1 + 1); old_icu2_mask = inb(IO_ICU2 + 1); +#endif /* APIC_IO */ save_idt_npxintr = idt[npx_intrno]; +#if defined(APIC_IO) + /** FIXME: try clrIoApicMaskBit( npx0_imask ); */ + INTRSET( old_apic_mask & ~(npx0_imask & 0xffff) ); +#else outb(IO_ICU1 + 1, old_icu1_mask & ~(IRQ_SLAVE | npx0_imask)); outb(IO_ICU2 + 1, old_icu2_mask & ~(npx0_imask >> 8)); +#endif /* APIC_IO */ idt[npx_intrno] = npx_idt_probeintr; enable_intr(); stop_emulating(); @@ -596,6 +659,11 @@ npxsave(addr) start_emulating(); npxproc = NULL; disable_intr(); +#if defined(APIC_IO) + apic_mask = INTRGET(); /* masks may have changed */ + INTRSET( (apic_mask & ~(npx0_imask & 0xffff)) | + (old_apic_mask & (npx0_imask & 0xffff))); +#else icu1_mask = inb(IO_ICU1 + 1); /* masks may have changed */ icu2_mask = inb(IO_ICU2 + 1); outb(IO_ICU1 + 1, @@ -603,6 +671,7 @@ npxsave(addr) outb(IO_ICU2 + 1, (icu2_mask & ~(npx0_imask >> 8)) | (old_icu2_mask & (npx0_imask >> 8))); +#endif /* APIC_IO */ idt[npx_intrno] = save_idt_npxintr; enable_intr(); /* back to usual state */ } diff --git a/sys/amd64/isa/vector.S b/sys/amd64/isa/vector.S index fa2fd42afb54..270ae208cc29 100644 --- a/sys/amd64/isa/vector.S +++ b/sys/amd64/isa/vector.S @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id$ + * $Id: vector.s,v 1.25 1997/02/22 09:37:23 peter Exp $ */ /* @@ -8,6 +8,12 @@ */ #include "opt_auto_eoi.h" +#include "opt_smp.h" + +#if defined(SMP) +#include <machine/smpasm.h> /* this includes <machine/apic.h> */ +#include <machine/smptests.h> /** TEST_CPUHITS */ +#endif /* SMP */ #include <i386/isa/icu.h> #ifdef PC98 @@ -22,6 +28,111 @@ #define ICU_IMR_OFFSET 1 /* IO_ICU{1,2} + 1 */ #endif + +#if defined(SMP) + +#define GET_MPLOCK call _get_mplock +#define REL_MPLOCK call _rel_mplock + +#else + +#define GET_MPLOCK /* NOP get Kernel Mutex */ +#define REL_MPLOCK /* NOP release mutex */ + +#endif /* SMP */ + + +#if defined(APIC_IO) + +#if defined(SMP) && defined(TEST_CPUHITS) + +#undef GET_MPLOCK +#define GET_MPLOCK \ + call _get_mplock ; \ + GETCPUID(%eax) ; \ + incl _cpuhits(,%eax,4) + +#endif /* SMP && TEST_CPUHITS */ + +#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) +#define IRQ_BIT(irq_num) (1 << (irq_num)) + +#define ENABLE_APIC \ + movl _apic_base, %eax ; \ + movl $0, APIC_EOI(%eax) + +#define ENABLE_ICU1 ENABLE_APIC +#define ENABLE_ICU1_AND_2 ENABLE_APIC + +#define MASK_IRQ(irq_num,icu) \ + orl $IRQ_BIT(irq_num),_imen ; /* set the mask bit */ \ + movl _io_apic_base,%ecx ; /* io apic addr */ \ + movl $REDTBL_IDX(irq_num),(%ecx) ; /* write the index */ \ + movl IOAPIC_WINDOW(%ecx),%eax ; /* current value */ \ + orl $IOART_INTMASK,%eax ; /* set the mask */ \ + movl %eax,IOAPIC_WINDOW(%ecx) ; /* new value */ + +#define UNMASK_IRQ(irq_num,icu) \ + andl $~IRQ_BIT(irq_num),_imen ; /* clear mask bit */ \ + movl _io_apic_base,%ecx ; /* io apic addr */ \ + movl $REDTBL_IDX(irq_num),(%ecx) ; /* write the index */ \ + movl IOAPIC_WINDOW(%ecx),%eax ; /* current value */ \ + andl $~IOART_INTMASK,%eax ; /* clear the mask */ \ + movl %eax,IOAPIC_WINDOW(%ecx) ; /* new value */ + +#define TEST_IRQ(irq_num,reg) \ + testl $IRQ_BIT(irq_num),%eax + +#define SET_IPENDING(irq_num) \ + orl $IRQ_BIT(irq_num),_ipending + +/* + * 'lazy masking' code submitted by: Bruce Evans <bde@zeta.org.au> + */ +#define MAYBE_MASK_IRQ(irq_num,icu) \ + testl $IRQ_BIT(irq_num),iactive ; /* lazy masking */ \ + je 1f ; /* NOT currently active */ \ + MASK_IRQ(irq_num,icu) ; \ + ENABLE_APIC ; \ + SET_IPENDING(irq_num) ; \ + REL_MPLOCK ; /* SMP release global lock */ \ + popl %es ; \ + popl %ds ; \ + popal ; \ + addl $4+4,%esp ; \ + iret ; \ +; \ + ALIGN_TEXT ; \ +1: ; \ + orl $IRQ_BIT(irq_num),iactive + +#define MAYBE_UNMASK_IRQ(irq_num,icu) \ + andl $~IRQ_BIT(irq_num),iactive ; \ + testl $IRQ_BIT(irq_num),_imen ; \ + je 3f ; \ + UNMASK_IRQ(irq_num,icu) ; \ +3: + +#else /* APIC_IO */ + +#define MASK_IRQ(irq_num,icu) \ + movb _imen + IRQ_BYTE(irq_num),%al ; \ + orb $IRQ_BIT(irq_num),%al ; \ + movb %al,_imen + IRQ_BYTE(irq_num) ; \ + outb %al,$icu+ICU_IMR_OFFSET + +#define UNMASK_IRQ(irq_num,icu) \ + movb _imen + IRQ_BYTE(irq_num),%al ; \ + andb $~IRQ_BIT(irq_num),%al ; \ + movb %al,_imen + IRQ_BYTE(irq_num) ; \ + outb %al,$icu+ICU_IMR_OFFSET + +#define TEST_IRQ(irq_num,reg) \ + testb $IRQ_BIT(irq_num),%reg + +#define SET_IPENDING(irq_num) \ + orb $IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num) + #define ICU_EOI 0x20 /* XXX - define elsewhere */ #define IRQ_BIT(irq_num) (1 << ((irq_num) % 8)) @@ -50,6 +161,15 @@ OUTB_ICU1 /* ... then first icu (if !AUTO_EOI_1) */ #endif +#define MAYBE_MASK_IRQ(irq_num,icu) \ + MASK_IRQ(irq_num,icu) + +#define MAYBE_UNMASK_IRQ(irq_num,icu) \ + UNMASK_IRQ(irq_num,icu) + +#endif /* APIC_IO */ + + #ifdef FAST_INTR_HANDLER_USES_ES #define ACTUALLY_PUSHED 1 #define MAYBE_MOVW_AX_ES movl %ax,%es @@ -125,6 +245,7 @@ IDTVEC(vec_name) ; \ movl %ax,%ds ; \ MAYBE_MOVW_AX_ES ; \ FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \ + GET_MPLOCK ; /* SMP Spin lock */ \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -138,6 +259,7 @@ IDTVEC(vec_name) ; \ jne 2f ; /* yes, maybe handle them */ \ 1: ; \ MEXITCOUNT ; \ + REL_MPLOCK ; /* SMP release global lock */ \ MAYBE_POPL_ES ; \ popl %ds ; \ popl %edx ; \ @@ -183,13 +305,11 @@ IDTVEC(vec_name) ; \ movl $KDSEL,%eax ; /* ... and reload with kernel's own ... */ \ movl %ax,%ds ; /* ... early for obsolete reasons */ \ movl %ax,%es ; \ - movb _imen + IRQ_BYTE(irq_num),%al ; \ - orb $IRQ_BIT(irq_num),%al ; \ - movb %al,_imen + IRQ_BYTE(irq_num) ; \ - outb %al,$icu+ICU_IMR_OFFSET ; \ + GET_MPLOCK ; /* SMP Spin lock */ \ + MAYBE_MASK_IRQ(irq_num,icu) ; \ enable_icus ; \ movl _cpl,%eax ; \ - testb $IRQ_BIT(irq_num),%reg ; \ + TEST_IRQ(irq_num,reg) ; \ jne 2f ; \ incb _intr_nesting_level ; \ __CONCAT(Xresume,irq_num): ; \ @@ -205,10 +325,7 @@ __CONCAT(Xresume,irq_num): ; \ sti ; \ call *_intr_handler + (irq_num) * 4 ; \ cli ; /* must unmask _imen and icu atomically */ \ - movb _imen + IRQ_BYTE(irq_num),%al ; \ - andb $~IRQ_BIT(irq_num),%al ; \ - movb %al,_imen + IRQ_BYTE(irq_num) ; \ - outb %al,$icu+ICU_IMR_OFFSET ; \ + MAYBE_UNMASK_IRQ(irq_num,icu) ; \ sti ; /* XXX _doreti repeats the cli/sti */ \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ @@ -218,13 +335,66 @@ __CONCAT(Xresume,irq_num): ; \ ALIGN_TEXT ; \ 2: ; \ /* XXX skip mcounting here to avoid double count */ \ - orb $IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num) ; \ + SET_IPENDING(irq_num) ; \ + REL_MPLOCK ; /* SMP release global lock */ \ popl %es ; \ popl %ds ; \ popal ; \ addl $4+4,%esp ; \ iret + +#if defined(APIC_IO) && defined(IPI_INTS) +/* + * A simple IPI_INTR() macro based on a heavily cut down FAST_INTR(). + * call it's handler, EOI and return. + */ +#define IPI_INTR(irq_num, vec_name) \ + .text ; \ + SUPERALIGN_TEXT ; \ +IDTVEC(vec_name) ; \ + pushl %eax ; /* save only call-used registers */ \ + pushl %ecx ; \ + pushl %edx ; \ + pushl %ds ; \ + MAYBE_PUSHL_ES ; \ + movl $KDSEL,%eax ; \ + movl %ax,%ds ; \ + MAYBE_MOVW_AX_ES ; \ + pushl _intr_unit + (irq_num) * 4 ; \ + call *_intr_handler + (irq_num) * 4 ; \ + ENABLE_APIC ; \ + addl $4,%esp ; \ + incl _cnt+V_INTR ; /* book-keeping can wait */ \ + movl _intr_countp + (irq_num) * 4,%eax ; \ + incl (%eax) ; \ + MAYBE_POPL_ES ; \ + popl %ds ; \ + popl %edx ; \ + popl %ecx ; \ + popl %eax ; \ + iret +#endif /* APIC_IO && IPI_INTS */ + +#if defined(XFAST_IPI32) + .text + SUPERALIGN_TEXT + .globl _Xfastipi32 +_Xfastipi32: + pushl %eax + movl %cr3, %eax + movl %eax, %cr3 + pushl %ds + movl $KDSEL,%eax + movl %ax,%ds + incl _ipihits + movl _apic_base, %eax + movl $0, APIC_EOI(%eax) + popl %ds + popl %eax + iret +#endif /* XFAST_IPI32 */ + MCOUNT_LABEL(bintr) FAST_INTR(0,fastintr0, ENABLE_ICU1) FAST_INTR(1,fastintr1, ENABLE_ICU1) @@ -242,6 +412,16 @@ MCOUNT_LABEL(bintr) FAST_INTR(13,fastintr13, ENABLE_ICU1_AND_2) FAST_INTR(14,fastintr14, ENABLE_ICU1_AND_2) FAST_INTR(15,fastintr15, ENABLE_ICU1_AND_2) +#if defined(APIC_IO) + FAST_INTR(16,fastintr16, ENABLE_ICU1_AND_2) + FAST_INTR(17,fastintr17, ENABLE_ICU1_AND_2) + FAST_INTR(18,fastintr18, ENABLE_ICU1_AND_2) + FAST_INTR(19,fastintr19, ENABLE_ICU1_AND_2) + FAST_INTR(20,fastintr20, ENABLE_ICU1_AND_2) + FAST_INTR(21,fastintr21, ENABLE_ICU1_AND_2) + FAST_INTR(22,fastintr22, ENABLE_ICU1_AND_2) + FAST_INTR(23,fastintr23, ENABLE_ICU1_AND_2) +#endif /* APIC_IO */ INTR(0,intr0, IO_ICU1, ENABLE_ICU1, al) INTR(1,intr1, IO_ICU1, ENABLE_ICU1, al) INTR(2,intr2, IO_ICU1, ENABLE_ICU1, al) @@ -258,6 +438,22 @@ MCOUNT_LABEL(bintr) INTR(13,intr13, IO_ICU2, ENABLE_ICU1_AND_2, ah) INTR(14,intr14, IO_ICU2, ENABLE_ICU1_AND_2, ah) INTR(15,intr15, IO_ICU2, ENABLE_ICU1_AND_2, ah) +#if defined(APIC_IO) + INTR(16,intr16, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(17,intr17, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(18,intr18, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(19,intr19, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(20,intr20, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(21,intr21, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(22,intr22, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(23,intr23, IO_ICU2, ENABLE_ICU1_AND_2, ah) +#if defined(IPI_INTS) + IPI_INTR(24, ipi24) + IPI_INTR(25, ipi25) + IPI_INTR(26, ipi26) + IPI_INTR(27, ipi27) +#endif /* IPI_INTS */ +#endif /* APIC_IO */ MCOUNT_LABEL(eintr) .data @@ -267,17 +463,74 @@ ihandlers: /* addresses of interrupt handlers */ .long Xresume4, Xresume5, Xresume6, Xresume7 .long Xresume8, Xresume9, Xresume10, Xresume11 .long Xresume12, Xresume13, Xresume14, Xresume15 +#if defined(APIC_IO) + .long Xresume16, Xresume17, Xresume18, Xresume19 + .long Xresume20, Xresume21, Xresume22, Xresume23 + .long 0, 0, 0, 0, swi_tty, swi_net, _softclock, swi_ast +#else .long swi_tty, swi_net, 0, 0, 0, 0, 0, 0 .long 0, 0, 0, 0, 0, 0, _softclock, swi_ast +#endif /* APIC_IO */ + imasks: /* masks for interrupt handlers */ .space NHWI*4 /* padding; HWI masks are elsewhere */ + +#if defined(APIC_IO) +#if defined(IPI_INTS) + /* these 4 IPI slots are counted as HARDWARE INTs, ie NHWI, above */ +#else + .long 0, 0, 0, 0 /* padding */ +#endif /* IPI_INTS */ + .long SWI_TTY_MASK, SWI_NET_MASK, SWI_CLOCK_MASK, SWI_AST_MASK +#else .long SWI_TTY_MASK, SWI_NET_MASK, 0, 0, 0, 0, 0, 0 .long 0, 0, 0, 0, 0, 0, SWI_CLOCK_MASK, SWI_AST_MASK +#endif /* APIC_IO */ + .globl _intr_nesting_level _intr_nesting_level: .byte 0 .space 3 +#if defined(APIC_IO) + + .globl _ivectors +_ivectors: + .long _Xintr0, _Xintr1, _Xintr2, _Xintr3 + .long _Xintr4, _Xintr5, _Xintr6, _Xintr7 + .long _Xintr8, _Xintr9, _Xintr10, _Xintr11 + .long _Xintr12, _Xintr13, _Xintr14, _Xintr15 + .long _Xintr16, _Xintr17, _Xintr18, _Xintr19 + .long _Xintr20, _Xintr21, _Xintr22, _Xintr23 +#if defined(IPI_INTS) + .long _Xipi24, _Xipi25, _Xipi26, _Xipi27 +#endif /* IPI_INTS */ + +/* active flag for lazy masking */ +iactive: + .long 0 + +#if defined(XFAST_IPI32) + .globl _ipihits +_ipihits: + .long 0 +#endif /* XFAST_IPI32 */ + +#if defined(TEST_CPUHITS) + .globl _cpuhits +_cpuhits: +#if !defined(NCPU) +/** + * FIXME: need a way to pass NCPU to .s files. + * NCPU currently defined in smp.h IF NOT defined in opt_smp.h. + */ +#define NCPU 4 +#endif /* NCPU */ + .space NCPU*4 +#endif /* TEST_CPUHITS */ + +#endif /* APIC_IO */ + /* * Interrupt counters and names. The format of these and the label names * must agree with what vmstat expects. The tables are indexed by device @@ -309,6 +562,22 @@ _intrnames: .asciz "stray irq13" .asciz "stray irq14" .asciz "stray irq15" +#if defined(APIC_IO) + .asciz "stray irq16" + .asciz "stray irq17" + .asciz "stray irq18" + .asciz "stray irq19" + .asciz "stray irq20" + .asciz "stray irq21" + .asciz "stray irq22" + .asciz "stray irq23" +#if defined(IPI_INTS) + .asciz "stray irq24" + .asciz "stray irq25" + .asciz "stray irq26" + .asciz "stray irq27" +#endif /* IPI_INTS */ +#endif /* APIC_IO */ _eintrnames: .text diff --git a/sys/amd64/isa/vector.s b/sys/amd64/isa/vector.s index fa2fd42afb54..270ae208cc29 100644 --- a/sys/amd64/isa/vector.s +++ b/sys/amd64/isa/vector.s @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id$ + * $Id: vector.s,v 1.25 1997/02/22 09:37:23 peter Exp $ */ /* @@ -8,6 +8,12 @@ */ #include "opt_auto_eoi.h" +#include "opt_smp.h" + +#if defined(SMP) +#include <machine/smpasm.h> /* this includes <machine/apic.h> */ +#include <machine/smptests.h> /** TEST_CPUHITS */ +#endif /* SMP */ #include <i386/isa/icu.h> #ifdef PC98 @@ -22,6 +28,111 @@ #define ICU_IMR_OFFSET 1 /* IO_ICU{1,2} + 1 */ #endif + +#if defined(SMP) + +#define GET_MPLOCK call _get_mplock +#define REL_MPLOCK call _rel_mplock + +#else + +#define GET_MPLOCK /* NOP get Kernel Mutex */ +#define REL_MPLOCK /* NOP release mutex */ + +#endif /* SMP */ + + +#if defined(APIC_IO) + +#if defined(SMP) && defined(TEST_CPUHITS) + +#undef GET_MPLOCK +#define GET_MPLOCK \ + call _get_mplock ; \ + GETCPUID(%eax) ; \ + incl _cpuhits(,%eax,4) + +#endif /* SMP && TEST_CPUHITS */ + +#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) +#define IRQ_BIT(irq_num) (1 << (irq_num)) + +#define ENABLE_APIC \ + movl _apic_base, %eax ; \ + movl $0, APIC_EOI(%eax) + +#define ENABLE_ICU1 ENABLE_APIC +#define ENABLE_ICU1_AND_2 ENABLE_APIC + +#define MASK_IRQ(irq_num,icu) \ + orl $IRQ_BIT(irq_num),_imen ; /* set the mask bit */ \ + movl _io_apic_base,%ecx ; /* io apic addr */ \ + movl $REDTBL_IDX(irq_num),(%ecx) ; /* write the index */ \ + movl IOAPIC_WINDOW(%ecx),%eax ; /* current value */ \ + orl $IOART_INTMASK,%eax ; /* set the mask */ \ + movl %eax,IOAPIC_WINDOW(%ecx) ; /* new value */ + +#define UNMASK_IRQ(irq_num,icu) \ + andl $~IRQ_BIT(irq_num),_imen ; /* clear mask bit */ \ + movl _io_apic_base,%ecx ; /* io apic addr */ \ + movl $REDTBL_IDX(irq_num),(%ecx) ; /* write the index */ \ + movl IOAPIC_WINDOW(%ecx),%eax ; /* current value */ \ + andl $~IOART_INTMASK,%eax ; /* clear the mask */ \ + movl %eax,IOAPIC_WINDOW(%ecx) ; /* new value */ + +#define TEST_IRQ(irq_num,reg) \ + testl $IRQ_BIT(irq_num),%eax + +#define SET_IPENDING(irq_num) \ + orl $IRQ_BIT(irq_num),_ipending + +/* + * 'lazy masking' code submitted by: Bruce Evans <bde@zeta.org.au> + */ +#define MAYBE_MASK_IRQ(irq_num,icu) \ + testl $IRQ_BIT(irq_num),iactive ; /* lazy masking */ \ + je 1f ; /* NOT currently active */ \ + MASK_IRQ(irq_num,icu) ; \ + ENABLE_APIC ; \ + SET_IPENDING(irq_num) ; \ + REL_MPLOCK ; /* SMP release global lock */ \ + popl %es ; \ + popl %ds ; \ + popal ; \ + addl $4+4,%esp ; \ + iret ; \ +; \ + ALIGN_TEXT ; \ +1: ; \ + orl $IRQ_BIT(irq_num),iactive + +#define MAYBE_UNMASK_IRQ(irq_num,icu) \ + andl $~IRQ_BIT(irq_num),iactive ; \ + testl $IRQ_BIT(irq_num),_imen ; \ + je 3f ; \ + UNMASK_IRQ(irq_num,icu) ; \ +3: + +#else /* APIC_IO */ + +#define MASK_IRQ(irq_num,icu) \ + movb _imen + IRQ_BYTE(irq_num),%al ; \ + orb $IRQ_BIT(irq_num),%al ; \ + movb %al,_imen + IRQ_BYTE(irq_num) ; \ + outb %al,$icu+ICU_IMR_OFFSET + +#define UNMASK_IRQ(irq_num,icu) \ + movb _imen + IRQ_BYTE(irq_num),%al ; \ + andb $~IRQ_BIT(irq_num),%al ; \ + movb %al,_imen + IRQ_BYTE(irq_num) ; \ + outb %al,$icu+ICU_IMR_OFFSET + +#define TEST_IRQ(irq_num,reg) \ + testb $IRQ_BIT(irq_num),%reg + +#define SET_IPENDING(irq_num) \ + orb $IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num) + #define ICU_EOI 0x20 /* XXX - define elsewhere */ #define IRQ_BIT(irq_num) (1 << ((irq_num) % 8)) @@ -50,6 +161,15 @@ OUTB_ICU1 /* ... then first icu (if !AUTO_EOI_1) */ #endif +#define MAYBE_MASK_IRQ(irq_num,icu) \ + MASK_IRQ(irq_num,icu) + +#define MAYBE_UNMASK_IRQ(irq_num,icu) \ + UNMASK_IRQ(irq_num,icu) + +#endif /* APIC_IO */ + + #ifdef FAST_INTR_HANDLER_USES_ES #define ACTUALLY_PUSHED 1 #define MAYBE_MOVW_AX_ES movl %ax,%es @@ -125,6 +245,7 @@ IDTVEC(vec_name) ; \ movl %ax,%ds ; \ MAYBE_MOVW_AX_ES ; \ FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \ + GET_MPLOCK ; /* SMP Spin lock */ \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -138,6 +259,7 @@ IDTVEC(vec_name) ; \ jne 2f ; /* yes, maybe handle them */ \ 1: ; \ MEXITCOUNT ; \ + REL_MPLOCK ; /* SMP release global lock */ \ MAYBE_POPL_ES ; \ popl %ds ; \ popl %edx ; \ @@ -183,13 +305,11 @@ IDTVEC(vec_name) ; \ movl $KDSEL,%eax ; /* ... and reload with kernel's own ... */ \ movl %ax,%ds ; /* ... early for obsolete reasons */ \ movl %ax,%es ; \ - movb _imen + IRQ_BYTE(irq_num),%al ; \ - orb $IRQ_BIT(irq_num),%al ; \ - movb %al,_imen + IRQ_BYTE(irq_num) ; \ - outb %al,$icu+ICU_IMR_OFFSET ; \ + GET_MPLOCK ; /* SMP Spin lock */ \ + MAYBE_MASK_IRQ(irq_num,icu) ; \ enable_icus ; \ movl _cpl,%eax ; \ - testb $IRQ_BIT(irq_num),%reg ; \ + TEST_IRQ(irq_num,reg) ; \ jne 2f ; \ incb _intr_nesting_level ; \ __CONCAT(Xresume,irq_num): ; \ @@ -205,10 +325,7 @@ __CONCAT(Xresume,irq_num): ; \ sti ; \ call *_intr_handler + (irq_num) * 4 ; \ cli ; /* must unmask _imen and icu atomically */ \ - movb _imen + IRQ_BYTE(irq_num),%al ; \ - andb $~IRQ_BIT(irq_num),%al ; \ - movb %al,_imen + IRQ_BYTE(irq_num) ; \ - outb %al,$icu+ICU_IMR_OFFSET ; \ + MAYBE_UNMASK_IRQ(irq_num,icu) ; \ sti ; /* XXX _doreti repeats the cli/sti */ \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ @@ -218,13 +335,66 @@ __CONCAT(Xresume,irq_num): ; \ ALIGN_TEXT ; \ 2: ; \ /* XXX skip mcounting here to avoid double count */ \ - orb $IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num) ; \ + SET_IPENDING(irq_num) ; \ + REL_MPLOCK ; /* SMP release global lock */ \ popl %es ; \ popl %ds ; \ popal ; \ addl $4+4,%esp ; \ iret + +#if defined(APIC_IO) && defined(IPI_INTS) +/* + * A simple IPI_INTR() macro based on a heavily cut down FAST_INTR(). + * call it's handler, EOI and return. + */ +#define IPI_INTR(irq_num, vec_name) \ + .text ; \ + SUPERALIGN_TEXT ; \ +IDTVEC(vec_name) ; \ + pushl %eax ; /* save only call-used registers */ \ + pushl %ecx ; \ + pushl %edx ; \ + pushl %ds ; \ + MAYBE_PUSHL_ES ; \ + movl $KDSEL,%eax ; \ + movl %ax,%ds ; \ + MAYBE_MOVW_AX_ES ; \ + pushl _intr_unit + (irq_num) * 4 ; \ + call *_intr_handler + (irq_num) * 4 ; \ + ENABLE_APIC ; \ + addl $4,%esp ; \ + incl _cnt+V_INTR ; /* book-keeping can wait */ \ + movl _intr_countp + (irq_num) * 4,%eax ; \ + incl (%eax) ; \ + MAYBE_POPL_ES ; \ + popl %ds ; \ + popl %edx ; \ + popl %ecx ; \ + popl %eax ; \ + iret +#endif /* APIC_IO && IPI_INTS */ + +#if defined(XFAST_IPI32) + .text + SUPERALIGN_TEXT + .globl _Xfastipi32 +_Xfastipi32: + pushl %eax + movl %cr3, %eax + movl %eax, %cr3 + pushl %ds + movl $KDSEL,%eax + movl %ax,%ds + incl _ipihits + movl _apic_base, %eax + movl $0, APIC_EOI(%eax) + popl %ds + popl %eax + iret +#endif /* XFAST_IPI32 */ + MCOUNT_LABEL(bintr) FAST_INTR(0,fastintr0, ENABLE_ICU1) FAST_INTR(1,fastintr1, ENABLE_ICU1) @@ -242,6 +412,16 @@ MCOUNT_LABEL(bintr) FAST_INTR(13,fastintr13, ENABLE_ICU1_AND_2) FAST_INTR(14,fastintr14, ENABLE_ICU1_AND_2) FAST_INTR(15,fastintr15, ENABLE_ICU1_AND_2) +#if defined(APIC_IO) + FAST_INTR(16,fastintr16, ENABLE_ICU1_AND_2) + FAST_INTR(17,fastintr17, ENABLE_ICU1_AND_2) + FAST_INTR(18,fastintr18, ENABLE_ICU1_AND_2) + FAST_INTR(19,fastintr19, ENABLE_ICU1_AND_2) + FAST_INTR(20,fastintr20, ENABLE_ICU1_AND_2) + FAST_INTR(21,fastintr21, ENABLE_ICU1_AND_2) + FAST_INTR(22,fastintr22, ENABLE_ICU1_AND_2) + FAST_INTR(23,fastintr23, ENABLE_ICU1_AND_2) +#endif /* APIC_IO */ INTR(0,intr0, IO_ICU1, ENABLE_ICU1, al) INTR(1,intr1, IO_ICU1, ENABLE_ICU1, al) INTR(2,intr2, IO_ICU1, ENABLE_ICU1, al) @@ -258,6 +438,22 @@ MCOUNT_LABEL(bintr) INTR(13,intr13, IO_ICU2, ENABLE_ICU1_AND_2, ah) INTR(14,intr14, IO_ICU2, ENABLE_ICU1_AND_2, ah) INTR(15,intr15, IO_ICU2, ENABLE_ICU1_AND_2, ah) +#if defined(APIC_IO) + INTR(16,intr16, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(17,intr17, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(18,intr18, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(19,intr19, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(20,intr20, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(21,intr21, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(22,intr22, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(23,intr23, IO_ICU2, ENABLE_ICU1_AND_2, ah) +#if defined(IPI_INTS) + IPI_INTR(24, ipi24) + IPI_INTR(25, ipi25) + IPI_INTR(26, ipi26) + IPI_INTR(27, ipi27) +#endif /* IPI_INTS */ +#endif /* APIC_IO */ MCOUNT_LABEL(eintr) .data @@ -267,17 +463,74 @@ ihandlers: /* addresses of interrupt handlers */ .long Xresume4, Xresume5, Xresume6, Xresume7 .long Xresume8, Xresume9, Xresume10, Xresume11 .long Xresume12, Xresume13, Xresume14, Xresume15 +#if defined(APIC_IO) + .long Xresume16, Xresume17, Xresume18, Xresume19 + .long Xresume20, Xresume21, Xresume22, Xresume23 + .long 0, 0, 0, 0, swi_tty, swi_net, _softclock, swi_ast +#else .long swi_tty, swi_net, 0, 0, 0, 0, 0, 0 .long 0, 0, 0, 0, 0, 0, _softclock, swi_ast +#endif /* APIC_IO */ + imasks: /* masks for interrupt handlers */ .space NHWI*4 /* padding; HWI masks are elsewhere */ + +#if defined(APIC_IO) +#if defined(IPI_INTS) + /* these 4 IPI slots are counted as HARDWARE INTs, ie NHWI, above */ +#else + .long 0, 0, 0, 0 /* padding */ +#endif /* IPI_INTS */ + .long SWI_TTY_MASK, SWI_NET_MASK, SWI_CLOCK_MASK, SWI_AST_MASK +#else .long SWI_TTY_MASK, SWI_NET_MASK, 0, 0, 0, 0, 0, 0 .long 0, 0, 0, 0, 0, 0, SWI_CLOCK_MASK, SWI_AST_MASK +#endif /* APIC_IO */ + .globl _intr_nesting_level _intr_nesting_level: .byte 0 .space 3 +#if defined(APIC_IO) + + .globl _ivectors +_ivectors: + .long _Xintr0, _Xintr1, _Xintr2, _Xintr3 + .long _Xintr4, _Xintr5, _Xintr6, _Xintr7 + .long _Xintr8, _Xintr9, _Xintr10, _Xintr11 + .long _Xintr12, _Xintr13, _Xintr14, _Xintr15 + .long _Xintr16, _Xintr17, _Xintr18, _Xintr19 + .long _Xintr20, _Xintr21, _Xintr22, _Xintr23 +#if defined(IPI_INTS) + .long _Xipi24, _Xipi25, _Xipi26, _Xipi27 +#endif /* IPI_INTS */ + +/* active flag for lazy masking */ +iactive: + .long 0 + +#if defined(XFAST_IPI32) + .globl _ipihits +_ipihits: + .long 0 +#endif /* XFAST_IPI32 */ + +#if defined(TEST_CPUHITS) + .globl _cpuhits +_cpuhits: +#if !defined(NCPU) +/** + * FIXME: need a way to pass NCPU to .s files. + * NCPU currently defined in smp.h IF NOT defined in opt_smp.h. + */ +#define NCPU 4 +#endif /* NCPU */ + .space NCPU*4 +#endif /* TEST_CPUHITS */ + +#endif /* APIC_IO */ + /* * Interrupt counters and names. The format of these and the label names * must agree with what vmstat expects. The tables are indexed by device @@ -309,6 +562,22 @@ _intrnames: .asciz "stray irq13" .asciz "stray irq14" .asciz "stray irq15" +#if defined(APIC_IO) + .asciz "stray irq16" + .asciz "stray irq17" + .asciz "stray irq18" + .asciz "stray irq19" + .asciz "stray irq20" + .asciz "stray irq21" + .asciz "stray irq22" + .asciz "stray irq23" +#if defined(IPI_INTS) + .asciz "stray irq24" + .asciz "stray irq25" + .asciz "stray irq26" + .asciz "stray irq27" +#endif /* IPI_INTS */ +#endif /* APIC_IO */ _eintrnames: .text diff --git a/sys/conf/NOTES b/sys/conf/NOTES index a3cf90cf8fa5..016a0246ec92 100644 --- a/sys/conf/NOTES +++ b/sys/conf/NOTES @@ -2,7 +2,7 @@ # LINT -- config file for checking all the sources, tries to pull in # as much of the source tree as it can. # -# $Id: LINT,v 1.326 1997/04/07 01:26:34 bde Exp $ +# $Id: LINT,v 1.327 1997/04/14 00:35:23 gibbs Exp $ # # NB: You probably don't want to try running a kernel built from this # file. Instead, you should start from GENERIC, and add options from @@ -62,6 +62,67 @@ config kernel root on wd0 dumps on wd0 ##################################################################### +# SMP OPTIONS: +# +# SMP enables building of a Symmetric MultiProcessor Kernel. +# APIC_IO enables the use of the IO APIC for Symmetric I/O. +# SMP_INVLTLB enables code to send inter-CPU TLB invalidation messages. +# NCPU sets the number of CPUs, defaults to 2. +# NBUS sets the number of busses, defaults to 4. +# NAPIC sets the number of IO APICs on the motherboard, defaults to 1. +# NINTR sets the total number of INTs provided by the motherboard. +# +# SMP_PRIVPAGES # BROKEN, DO NOT use! (architecture problem) +# SMP_AUTOSTART # BROKEN, DO NOT use! (bug or race somewhere) +# +# SMP_TIMER_NC is for motherboards that claim 8254 connectivity to the IO APIC, +# when in fact it is NOT connected. +# +# Notes: +# +# An SMP kernel will ONLY run on an Intel MP spec. qualified motherboard. +# +# Be sure to disable 'cpu "I386_CPU"' && 'cpu "I486_CPU"' for SMP kernels. +# +# Check the 'Rogue SMP hardware' section to see if additional options +# are required by your hardware. +# + +# Mandatory: +options SMP # Symmetric MultiProcessor Kernel + +# Recommended: +options APIC_IO # Symmetric (APIC) I/O +options SMP_INVLTLB # invalidate TLB IPIs + +# Optional, these are the defaults: +#options NCPU=2 # number of CPUs +#options NBUS=4 # number of busses +#options NAPIC=1 # number of IO APICs +#options NINTR=24 # number of INTs + +# Currently unusable: +#options SMP_PRIVPAGES # BROKEN: architecture problem +#options SMP_AUTOSTART # BROKEN: bug or race somewhere + +# +# Rogue SMP hardware: +# + +# Tyan Tomcat II: +#options SMP_TIMER_NC # 8254 NOT connected to APIC + +# SuperMicro P6DNE: +#options SMP_TIMER_NC # 8254 NOT connected to APIC + +# Bridged PCI cards: +# +# The MP tables of most of the current generation MP motherboards +# do NOT properly support bridged PCI cards. To use one of these +# cards you should refer to ??? + + +##################################################################### # CPU OPTIONS # diff --git a/sys/conf/files b/sys/conf/files index aded07eb0378..74eab7354968 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -45,6 +45,7 @@ kern/imgact_gzip.c optional gzip kern/imgact_shell.c standard kern/inflate.c optional gzip kern/init_main.c standard +kern/init_smp.c optional smp kern/init_sysent.c standard kern/init_sysvec.c standard kern/kern_acct.c standard diff --git a/sys/conf/files.i386 b/sys/conf/files.i386 index e51cc4c05540..320e6c0a7f32 100644 --- a/sys/conf/files.i386 +++ b/sys/conf/files.i386 @@ -1,7 +1,7 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # -# $Id: files.i386,v 1.158 1997/04/03 05:57:16 gibbs Exp $ +# $Id: files.i386,v 1.159 1997/04/14 00:35:25 gibbs Exp $ # aic7xxx_asm optional ahc device-driver \ dependency "$S/dev/aic7xxx/*.[chyl]" \ @@ -55,6 +55,10 @@ i386/i386/machdep.c standard i386/i386/math_emulate.c optional math_emulate i386/i386/mem.c standard i386/i386/microtime.s standard +i386/i386/mp_machdep.c optional smp +i386/i386/mpapic.c optional smp +i386/i386/mpboot.s optional smp +i386/i386/mplock.s optional smp i386/i386/perfmon.c optional perfmon profiling-routine i386/i386/perfmon.c optional perfmon i386/i386/pmap.c standard diff --git a/sys/conf/options.i386 b/sys/conf/options.i386 index de07655f0d93..57ba3edb0ac7 100644 --- a/sys/conf/options.i386 +++ b/sys/conf/options.i386 @@ -1,4 +1,4 @@ -# $Id: options.i386,v 1.38 1997/03/22 18:51:32 kato Exp $ +# $Id: options.i386,v 1.39 1997/04/05 13:21:08 bde Exp $ BOUNCEPAGES opt_bounce.h USER_LDT MATH_EMULATE opt_math_emulate.h @@ -28,6 +28,23 @@ PCVT_FREEBSD opt_pcvt.h PCVT_SCANSET opt_pcvt.h XSERVER opt_pcvt.h +SMP opt_smp.h +APIC_IO opt_smp.h +NCPU opt_smp.h +NBUS opt_smp.h +NAPIC opt_smp.h +NINTR opt_smp.h +SMP_TIMER_NC opt_smp.h + +# Should be working. When one cpu flushes it's TLB, it's propagated to all. +SMP_INVLTLB opt_smp_invltlb.h + +# These three are known to be broken, don't enable them. +SMP_PRIVPAGES opt_smp_privpages.h +SMP_AUTOSTART opt_smp_autostart.h + +SERIAL_DEBUG opt_serial.h + AHC_TAGENABLE opt_aic7xxx.h AHC_SCBPAGING_ENABLE opt_aic7xxx.h AHC_ALLOW_MEMIO opt_aic7xxx.h diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c index 8c1bc7dbc19b..96a71e841cf0 100644 --- a/sys/dev/pci/pci.c +++ b/sys/dev/pci/pci.c @@ -1,6 +1,6 @@ /************************************************************************** ** -** $Id: pci.c,v 1.68 1997/03/25 19:12:08 se Exp $ +** $Id: pci.c,v 1.69 1997/04/23 19:43:20 se Exp $ ** ** General subroutines for the PCI bus. ** pci_configure () @@ -46,6 +46,8 @@ **======================================================== */ +#include "opt_smp.h" + #include <sys/param.h> #include <sys/systm.h> #include <sys/malloc.h> @@ -70,9 +72,6 @@ #include <pci/pcibus.h> #include <pci/pci_ioctl.h> -#define PCI_MAX_IRQ (16) - - /*======================================================== ** ** Structs and Functions @@ -403,6 +402,10 @@ static void pci_attach (int bus, int dev, int func, u_char reg; u_char pciint; int irq; +#if defined(APIC_IO) + u_char airq = 0xff; + u_char rirq = 0xff; +#endif /* APIC_IO */ pcici_t tag = pcibus->pb_tag (bus, dev, func); /* @@ -437,14 +440,33 @@ static void pci_attach (int bus, int dev, int func, ** and we cannot bind the pci interrupt. */ +#if defined(APIC_IO) + if (irq && (irq != 0xff)) { + airq = get_pci_apic_irq (bus, dev, pciint); + if (airq != 0xff) { /* APIC IRQ exists */ + rirq = irq; /* 're-directed' IRQ */ + irq = airq; /* use APIC IRQ */ + } + printf ("%d", irq); + } +#else if (irq && (irq != 0xff)) printf ("%d", irq); +#endif /* APIC_IO */ else printf ("??"); }; printf (" on pci%d:%d:%d\n", bus, dev, func); +#if defined(APIC_IO) + if (airq != 0xff) { /* APIC IRQ exists */ + data = PCI_INTERRUPT_LINE_INSERT(data, airq); + pci_conf_write (tag, PCI_INTERRUPT_REG, data); + undirect_pci_irq (rirq); /* free for ISA card */ + } +#endif /* APIC_IO */ + /* ** Read the current mapping, ** and update the pcicb fields. diff --git a/sys/dev/sio/sio.c b/sys/dev/sio/sio.c index 7d5e78db1be2..54420e863661 100644 --- a/sys/dev/sio/sio.c +++ b/sys/dev/sio/sio.c @@ -31,12 +31,13 @@ * SUCH DAMAGE. * * from: @(#)com.c 7.5 (Berkeley) 5/16/91 - * $Id: sio.c,v 1.162 1997/04/05 13:11:27 bde Exp $ + * $Id: sio.c,v 1.163 1997/04/18 18:28:09 bde Exp $ */ #include "opt_comconsole.h" #include "opt_ddb.h" #include "opt_sio.h" +#include "opt_smp.h" #include "sio.h" /* @@ -82,6 +83,16 @@ #include <pccard/slot.h> #endif +#if defined(APIC_IO) +/* + * INTs are masked in the (global) IO APIC, + * but the IRR register is in each LOCAL APIC, + * so we HAVE to unmask the INT to be able to "see INT pending" + * BUT how do we clear them??? + */ +#define isa_irq_pending icu_irq_pending +#endif /* APIC_IO */ + #define LOTS_OF_EVENTS 64 /* helps separate urgent events from input */ #define RB_I_HIGH_WATER (TTYHOG - 2 * RS_IBUFSIZE) #define RS_IBUFSIZE 256 diff --git a/sys/i386/conf/LINT b/sys/i386/conf/LINT index a3cf90cf8fa5..016a0246ec92 100644 --- a/sys/i386/conf/LINT +++ b/sys/i386/conf/LINT @@ -2,7 +2,7 @@ # LINT -- config file for checking all the sources, tries to pull in # as much of the source tree as it can. # -# $Id: LINT,v 1.326 1997/04/07 01:26:34 bde Exp $ +# $Id: LINT,v 1.327 1997/04/14 00:35:23 gibbs Exp $ # # NB: You probably don't want to try running a kernel built from this # file. Instead, you should start from GENERIC, and add options from @@ -62,6 +62,67 @@ config kernel root on wd0 dumps on wd0 ##################################################################### +# SMP OPTIONS: +# +# SMP enables building of a Symmetric MultiProcessor Kernel. +# APIC_IO enables the use of the IO APIC for Symmetric I/O. +# SMP_INVLTLB enables code to send inter-CPU TLB invalidation messages. +# NCPU sets the number of CPUs, defaults to 2. +# NBUS sets the number of busses, defaults to 4. +# NAPIC sets the number of IO APICs on the motherboard, defaults to 1. +# NINTR sets the total number of INTs provided by the motherboard. +# +# SMP_PRIVPAGES # BROKEN, DO NOT use! (architecture problem) +# SMP_AUTOSTART # BROKEN, DO NOT use! (bug or race somewhere) +# +# SMP_TIMER_NC is for motherboards that claim 8254 connectivity to the IO APIC, +# when in fact it is NOT connected. +# +# Notes: +# +# An SMP kernel will ONLY run on an Intel MP spec. qualified motherboard. +# +# Be sure to disable 'cpu "I386_CPU"' && 'cpu "I486_CPU"' for SMP kernels. +# +# Check the 'Rogue SMP hardware' section to see if additional options +# are required by your hardware. +# + +# Mandatory: +options SMP # Symmetric MultiProcessor Kernel + +# Recommended: +options APIC_IO # Symmetric (APIC) I/O +options SMP_INVLTLB # invalidate TLB IPIs + +# Optional, these are the defaults: +#options NCPU=2 # number of CPUs +#options NBUS=4 # number of busses +#options NAPIC=1 # number of IO APICs +#options NINTR=24 # number of INTs + +# Currently unusable: +#options SMP_PRIVPAGES # BROKEN: architecture problem +#options SMP_AUTOSTART # BROKEN: bug or race somewhere + +# +# Rogue SMP hardware: +# + +# Tyan Tomcat II: +#options SMP_TIMER_NC # 8254 NOT connected to APIC + +# SuperMicro P6DNE: +#options SMP_TIMER_NC # 8254 NOT connected to APIC + +# Bridged PCI cards: +# +# The MP tables of most of the current generation MP motherboards +# do NOT properly support bridged PCI cards. To use one of these +# cards you should refer to ??? + + +##################################################################### # CPU OPTIONS # diff --git a/sys/i386/conf/NOTES b/sys/i386/conf/NOTES index a3cf90cf8fa5..016a0246ec92 100644 --- a/sys/i386/conf/NOTES +++ b/sys/i386/conf/NOTES @@ -2,7 +2,7 @@ # LINT -- config file for checking all the sources, tries to pull in # as much of the source tree as it can. # -# $Id: LINT,v 1.326 1997/04/07 01:26:34 bde Exp $ +# $Id: LINT,v 1.327 1997/04/14 00:35:23 gibbs Exp $ # # NB: You probably don't want to try running a kernel built from this # file. Instead, you should start from GENERIC, and add options from @@ -62,6 +62,67 @@ config kernel root on wd0 dumps on wd0 ##################################################################### +# SMP OPTIONS: +# +# SMP enables building of a Symmetric MultiProcessor Kernel. +# APIC_IO enables the use of the IO APIC for Symmetric I/O. +# SMP_INVLTLB enables code to send inter-CPU TLB invalidation messages. +# NCPU sets the number of CPUs, defaults to 2. +# NBUS sets the number of busses, defaults to 4. +# NAPIC sets the number of IO APICs on the motherboard, defaults to 1. +# NINTR sets the total number of INTs provided by the motherboard. +# +# SMP_PRIVPAGES # BROKEN, DO NOT use! (architecture problem) +# SMP_AUTOSTART # BROKEN, DO NOT use! (bug or race somewhere) +# +# SMP_TIMER_NC is for motherboards that claim 8254 connectivity to the IO APIC, +# when in fact it is NOT connected. +# +# Notes: +# +# An SMP kernel will ONLY run on an Intel MP spec. qualified motherboard. +# +# Be sure to disable 'cpu "I386_CPU"' && 'cpu "I486_CPU"' for SMP kernels. +# +# Check the 'Rogue SMP hardware' section to see if additional options +# are required by your hardware. +# + +# Mandatory: +options SMP # Symmetric MultiProcessor Kernel + +# Recommended: +options APIC_IO # Symmetric (APIC) I/O +options SMP_INVLTLB # invalidate TLB IPIs + +# Optional, these are the defaults: +#options NCPU=2 # number of CPUs +#options NBUS=4 # number of busses +#options NAPIC=1 # number of IO APICs +#options NINTR=24 # number of INTs + +# Currently unusable: +#options SMP_PRIVPAGES # BROKEN: architecture problem +#options SMP_AUTOSTART # BROKEN: bug or race somewhere + +# +# Rogue SMP hardware: +# + +# Tyan Tomcat II: +#options SMP_TIMER_NC # 8254 NOT connected to APIC + +# SuperMicro P6DNE: +#options SMP_TIMER_NC # 8254 NOT connected to APIC + +# Bridged PCI cards: +# +# The MP tables of most of the current generation MP motherboards +# do NOT properly support bridged PCI cards. To use one of these +# cards you should refer to ??? + + +##################################################################### # CPU OPTIONS # diff --git a/sys/i386/conf/files.i386 b/sys/i386/conf/files.i386 index e51cc4c05540..320e6c0a7f32 100644 --- a/sys/i386/conf/files.i386 +++ b/sys/i386/conf/files.i386 @@ -1,7 +1,7 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # -# $Id: files.i386,v 1.158 1997/04/03 05:57:16 gibbs Exp $ +# $Id: files.i386,v 1.159 1997/04/14 00:35:25 gibbs Exp $ # aic7xxx_asm optional ahc device-driver \ dependency "$S/dev/aic7xxx/*.[chyl]" \ @@ -55,6 +55,10 @@ i386/i386/machdep.c standard i386/i386/math_emulate.c optional math_emulate i386/i386/mem.c standard i386/i386/microtime.s standard +i386/i386/mp_machdep.c optional smp +i386/i386/mpapic.c optional smp +i386/i386/mpboot.s optional smp +i386/i386/mplock.s optional smp i386/i386/perfmon.c optional perfmon profiling-routine i386/i386/perfmon.c optional perfmon i386/i386/pmap.c standard diff --git a/sys/i386/conf/options.i386 b/sys/i386/conf/options.i386 index de07655f0d93..57ba3edb0ac7 100644 --- a/sys/i386/conf/options.i386 +++ b/sys/i386/conf/options.i386 @@ -1,4 +1,4 @@ -# $Id: options.i386,v 1.38 1997/03/22 18:51:32 kato Exp $ +# $Id: options.i386,v 1.39 1997/04/05 13:21:08 bde Exp $ BOUNCEPAGES opt_bounce.h USER_LDT MATH_EMULATE opt_math_emulate.h @@ -28,6 +28,23 @@ PCVT_FREEBSD opt_pcvt.h PCVT_SCANSET opt_pcvt.h XSERVER opt_pcvt.h +SMP opt_smp.h +APIC_IO opt_smp.h +NCPU opt_smp.h +NBUS opt_smp.h +NAPIC opt_smp.h +NINTR opt_smp.h +SMP_TIMER_NC opt_smp.h + +# Should be working. When one cpu flushes it's TLB, it's propagated to all. +SMP_INVLTLB opt_smp_invltlb.h + +# These three are known to be broken, don't enable them. +SMP_PRIVPAGES opt_smp_privpages.h +SMP_AUTOSTART opt_smp_autostart.h + +SERIAL_DEBUG opt_serial.h + AHC_TAGENABLE opt_aic7xxx.h AHC_SCBPAGING_ENABLE opt_aic7xxx.h AHC_ALLOW_MEMIO opt_aic7xxx.h diff --git a/sys/i386/i386/autoconf.c b/sys/i386/i386/autoconf.c index 5d297bfa953f..70e0b137dfee 100644 --- a/sys/i386/i386/autoconf.c +++ b/sys/i386/i386/autoconf.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)autoconf.c 7.1 (Berkeley) 5/9/91 - * $Id$ + * $Id: autoconf.c,v 1.64 1997/02/22 09:32:08 peter Exp $ */ /* @@ -45,6 +45,8 @@ * devices are determined (from possibilities mentioned in ioconf.c), * and the drivers are initialized. */ +#include "opt_smp.h" + #include <sys/param.h> #include <sys/systm.h> #include <sys/buf.h> @@ -61,6 +63,10 @@ #include <machine/md_var.h> #include <i386/isa/icu.h> /* For interrupts */ +#if defined(APIC_IO) +#include <machine/smp.h> +#endif /* APIC_IO */ + #include "isa.h" #if NISA > 0 #include <i386/isa/isa_device.h> @@ -158,8 +164,18 @@ configure(dummy) configure_start(); /* Allow all routines to decide for themselves if they want intrs */ +#if defined(APIC_IO) + configure_local_apic(); + enable_intr(); +#else enable_intr(); INTREN(IRQ_SLAVE); +#endif /* APIC_IO */ + +#if NCRD > 0 + /* Before isa_configure to avoid ISA drivers finding our cards */ + pccard_configure(); +#endif #if NEISA > 0 eisa_configure(); diff --git a/sys/i386/i386/db_interface.c b/sys/i386/i386/db_interface.c index c286027d53bd..adee9ff7fe47 100644 --- a/sys/i386/i386/db_interface.c +++ b/sys/i386/i386/db_interface.c @@ -23,7 +23,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: db_interface.c,v 1.26 1997/02/22 09:32:14 peter Exp $ + * $Id: db_interface.c,v 1.27 1997/03/28 12:56:07 bde Exp $ */ /* @@ -265,6 +265,8 @@ Debugger(msg) if (!in_Debugger) { in_Debugger = 1; + printf("CR0=%08x",rcr0()); + printf("CR3=%08x",rcr3()); db_printf("Debugger(\"%s\")\n", msg); breakpoint(); in_Debugger = 0; diff --git a/sys/i386/i386/db_trace.c b/sys/i386/i386/db_trace.c index 546df763b208..f8a934cf05ed 100644 --- a/sys/i386/i386/db_trace.c +++ b/sys/i386/i386/db_trace.c @@ -23,7 +23,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: db_trace.c,v 1.19 1997/02/28 12:36:18 bde Exp $ + * $Id: db_trace.c,v 1.20 1997/03/01 05:44:09 ache Exp $ */ #include <sys/param.h> @@ -199,6 +199,8 @@ db_nextframe(fp, ip) db_printf( "--- trap %#n, eip = %#n, esp = %#n, ebp = %#n ---\n", tf->tf_trapno, eip, esp, ebp); + db_printf("--- curproc = 0x%x, pid = %d ---\n", + curproc, curproc ? curproc->p_pid : 0); } break; case SYSCALL: @@ -208,6 +210,8 @@ db_nextframe(fp, ip) db_printf( "--- syscall %#n, eip = %#n, esp = %#n, ebp = %#n ---\n", tf->tf_eax, eip, esp, ebp); + db_printf("--- curproc = 0x%x, pid = %d ---\n", + curproc, curproc ? curproc->p_pid : 0); } break; case INTERRUPT: @@ -218,6 +222,8 @@ db_nextframe(fp, ip) db_printf( "--- interrupt, eip = %#n, esp = %#n, ebp = %#n ---\n", eip, esp, ebp); + db_printf("--- curproc = 0x%x, pid = %d ---\n", + curproc, curproc ? curproc->p_pid : 0); } break; default: diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s index 0ab3b33952e2..2705c08efcfe 100644 --- a/sys/i386/i386/exception.s +++ b/sys/i386/i386/exception.s @@ -30,9 +30,10 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: exception.s,v 1.23 1997/04/07 11:42:09 peter Exp $ + * $Id: exception.s,v 1.24 1997/04/13 16:58:08 bde Exp $ */ +#include "opt_smp.h" #include "npx.h" /* NNPX */ #include "assym.s" /* system defines */ #include <sys/errno.h> /* error return codes */ @@ -42,6 +43,7 @@ #include <sys/syscall.h> /* syscall numbers */ #include <machine/asmacros.h> /* miscellaneous macros */ #include <sys/cdefs.h> /* CPP macros */ +#include <machine/smpasm.h> #define KDSEL 0x10 /* kernel data selector */ #define SEL_RPL_MASK 0x0003 @@ -138,6 +140,9 @@ IDTVEC(fpu) movl _cpl,%eax pushl %eax pushl $0 /* dummy unit to finish building intr frame */ +#ifdef SMP + call _get_mplock +#endif /* SMP */ incl _cnt+V_TRAP orl $SWI_AST_MASK,%eax movl %eax,_cpl @@ -163,6 +168,9 @@ alltraps_with_regs_pushed: movl %ax,%es FAKE_MCOUNT(12*4(%esp)) calltrap: +#ifdef SMP + call _get_mplock +#endif /* SMP */ FAKE_MCOUNT(_btrap) /* init "from" _btrap -> calltrap */ incl _cnt+V_TRAP orl $SWI_AST_MASK,_cpl @@ -212,6 +220,9 @@ IDTVEC(syscall) movl %eax,TF_EFLAGS(%esp) movl $7,TF_ERR(%esp) /* sizeof "lcall 7,0" */ FAKE_MCOUNT(12*4(%esp)) +#ifdef SMP + call _get_mplock +#endif /* SMP */ incl _cnt+V_SYSCALL movl $SWI_AST_MASK,_cpl call _syscall @@ -238,6 +249,9 @@ IDTVEC(int0x80_syscall) movl %ax,%es movl $2,TF_ERR(%esp) /* sizeof "int 0x80" */ FAKE_MCOUNT(12*4(%esp)) +#ifdef SMP + call _get_mplock +#endif /* SMP */ incl _cnt+V_SYSCALL movl $SWI_AST_MASK,_cpl call _syscall @@ -254,7 +268,13 @@ ENTRY(fork_trampoline) movl $SWI_AST_MASK,_cpl call _splz +#if defined(SMP) + GETCPUID(%eax) + leal _SMPruntime(,%eax,8), %eax + pushl %eax +#else pushl $_runtime +#endif call _microtime popl %eax diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c index c1d3b4c1b752..e15535b82786 100644 --- a/sys/i386/i386/genassym.c +++ b/sys/i386/i386/genassym.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)genassym.c 5.11 (Berkeley) 5/10/91 - * $Id: genassym.c,v 1.43 1997/04/07 06:45:11 peter Exp $ + * $Id: genassym.c,v 1.44 1997/04/07 07:15:48 peter Exp $ */ #include <stdio.h> @@ -102,6 +102,8 @@ main() printf("#define\tP_WCHAN %p\n", &p->p_wchan); printf("#define\tP_FLAG %p\n", &p->p_flag); printf("#define\tP_PID %p\n", &p->p_pid); + printf("#define\tP_ONCPU %p\n", &p->p_oncpu); + printf("#define\tP_LASTCPU %p\n", &p->p_lastcpu); printf("#define\tSSLEEP %d\n", SSLEEP); printf("#define\tSRUN %d\n", SRUN); printf("#define\tV_TRAP %p\n", &vm->v_trap); @@ -131,6 +133,7 @@ main() printf("#define\tPCB_EIP %p\n", &pcb->pcb_eip); printf("#define\tTSS_ESP0 %p\n", &tss->tss_esp0); printf("#define\tPCB_USERLDT %p\n", &pcb->pcb_ldt); + printf("#define\tPCB_MPNEST %p\n", &pcb->pcb_mpnest); printf("#define\tU_PROF %p\n", &up->u_stats.p_prof); printf("#define\tU_PROFSCALE %p\n", &up->u_stats.p_prof.pr_scale); printf("#define\tPR_BASE %p\n", &uprof->pr_base); diff --git a/sys/i386/i386/identcpu.c b/sys/i386/i386/identcpu.c index 5defd2aeec00..21a62de7811a 100644 --- a/sys/i386/i386/identcpu.c +++ b/sys/i386/i386/identcpu.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * from: Id: machdep.c,v 1.193 1996/06/18 01:22:04 bde Exp - * $Id: identcpu.c,v 1.15 1997/04/22 06:55:23 jdp Exp $ + * $Id: identcpu.c,v 1.16 1997/04/26 04:08:45 kato Exp $ */ #include "opt_cpu.h" @@ -307,17 +307,21 @@ printcpuinfo(void) #endif #if defined(I586_CPU) case CPUCLASS_586: +#ifndef SMP printf("%d.%02d-MHz ", (i586_ctr_freq + 4999) / 1000000, ((i586_ctr_freq + 4999) / 10000) % 100); +#endif printf("586"); break; #endif #if defined(I686_CPU) case CPUCLASS_686: +#ifndef SMP printf("%d.%02d-MHz ", (i586_ctr_freq + 4999) / 1000000, ((i586_ctr_freq + 4999) / 10000) % 100); +#endif printf("686"); break; #endif diff --git a/sys/i386/i386/locore.s b/sys/i386/i386/locore.s index 63c11123f222..05ce518c0b7b 100644 --- a/sys/i386/i386/locore.s +++ b/sys/i386/i386/locore.s @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)locore.s 7.3 (Berkeley) 5/13/91 - * $Id: locore.s,v 1.84 1997/04/07 08:38:19 peter Exp $ + * $Id: locore.s,v 1.85 1997/04/22 06:55:24 jdp Exp $ * * originally from: locore.s, by William F. Jolitz * @@ -47,6 +47,9 @@ #include "opt_cpu.h" #include "opt_ddb.h" #include "opt_userconfig.h" +#include "opt_smp.h" +#include "opt_smp_privpages.h" +#include "opt_serial.h" #include <sys/errno.h> #include <sys/syscall.h> @@ -58,6 +61,10 @@ #include <machine/pmap.h> #include <machine/specialreg.h> +#if defined(SMP) && defined(SMP_PRIVPAGES) +#include <machine/apic.h> +#endif /* SMP && PRIVPAGES */ + #include "assym.s" /* @@ -111,6 +118,15 @@ _bootinfo: .space BOOTINFO_SIZE /* bootinfo that we can handle */ _KERNend: .long 0 /* phys addr end of kernel (just after bss) */ physfree: .long 0 /* phys addr of next free page */ +#if defined(SMP) && defined(SMP_PRIVPAGES) +cpu0pp: .long 0 /* phys addr cpu0 private pg */ +cpu0pt: .long 0 /* phys addr cpu0 private pt */ + + .globl _cpu0prvpage,_cpu0prvpt +_cpu0prvpage: .long 0 /* relocated version */ +_cpu0prvpt: .long 0 /* relocated version */ +#endif /* SMP && PRIVPAGES */ + .globl _IdlePTD _IdlePTD: .long 0 /* phys addr of kernel PTD */ @@ -752,6 +768,20 @@ over_symalloc: addl $KERNBASE, %esi movl %esi, R(_proc0paddr) +#if defined(SMP) && defined(SMP_PRIVPAGES) +/* Allocate cpu0's private data page */ + ALLOCPAGES(1) + movl %esi,R(cpu0pp) + addl $KERNBASE, %esi + movl %esi, R(_cpu0prvpage) /* relocated to KVM space */ + +/* Allocate cpu0's private page table for mapping priv page, apic, etc */ + ALLOCPAGES(1) + movl %esi,R(cpu0pt) + addl $KERNBASE, %esi + movl %esi, R(_cpu0prvpt) /* relocated to KVM space */ +#endif /* SMP && SMP_PRIVPAGES */ + /* Map read-only from zero to the end of the kernel text section */ xorl %eax, %eax #ifdef BDE_DEBUGGER @@ -799,6 +829,42 @@ map_read_write: movl $ISA_HOLE_LENGTH>>PAGE_SHIFT, %ecx fillkptphys($PG_RW) +#if defined(SMP) && defined(SMP_PRIVPAGES) +/* Map cpu0's private page into global KVM */ + movl R(cpu0pp), %eax + movl $1, %ecx + fillkptphys($PG_RW) + +/* Map cpu0's private page table into global KVM */ + movl R(cpu0pt), %eax + movl $1, %ecx + fillkptphys($PG_RW) + +/* Map the private page into the private page table (4K @ 0xff80000) */ + movl R(cpu0pp), %eax + movl $0, %ebx /* pte offset = 0 */ + movl $1, %ecx /* one private page coming right up */ + fillkpt(R(cpu0pt), $PG_RW) + +/* Map the default Local APIC address (4K @ 0xff801000) */ + movl $DEFAULT_APIC_BASE, %eax /* XXX just testing.. */ + movl $1, %ebx /* pte offset = 1 */ + movl $1, %ecx /* Bing! Local APIC appears */ + fillkpt(R(cpu0pt), $PG_RW|PG_N) + +/* Map the default IO APIC address (4K @ 0xff802000) */ + movl $DEFAULT_IO_APIC_BASE, %eax /* XXX just testing.. */ + movl $2, %ebx /* pte offset = 2 */ + movl $1, %ecx /* Bing! Local APIC appears */ + fillkpt(R(cpu0pt), $PG_RW|PG_N) + +/* ... and put the page table in the pde. */ + movl R(cpu0pt), %eax + movl $MPPTDI, %ebx + movl $1, %ecx + fillkpt(R(_IdlePTD), $PG_RW) +#endif /* SMP && SMP_PRIVPAGES */ + /* install a pde for temporary double map of bottom of VA */ movl R(_KPTphys), %eax xorl %ebx, %ebx diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index 9ed0b1d42d0f..d3b73299c37f 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91 - * $Id: machdep.c,v 1.237 1997/04/13 04:07:24 dyson Exp $ + * $Id: machdep.c,v 1.238 1997/04/22 06:55:26 jdp Exp $ */ #include "npx.h" @@ -44,6 +44,7 @@ #include "opt_bounce.h" #include "opt_machdep.h" #include "opt_perfmon.h" +#include "opt_smp.h" #include "opt_userconfig.h" #include <sys/param.h> @@ -107,6 +108,9 @@ #include <machine/cons.h> #include <machine/bootinfo.h> #include <machine/md_var.h> +#ifdef SMP +#include <machine/smp.h> +#endif #ifdef PERFMON #include <machine/perfmon.h> #endif @@ -209,6 +213,9 @@ cpu_startup(dummy) * Good {morning,afternoon,evening,night}. */ printf(version); +#ifdef SMP + mp_announce(); +#endif earlysetcpuclass(); startrtclock(); printcpuinfo(); @@ -344,6 +351,23 @@ again: u_map = kmem_suballoc(kernel_map, &minaddr, &maxaddr, (maxproc*UPAGES*PAGE_SIZE), FALSE); +#if defined(SMP) && defined(SMP_PRIVPAGES) + /* Per-cpu pages.. (the story so far is... subject to change) + * ========= For the per-cpu data page ======== + * 1 private data page + * 1 PDE (per-cpu PTD entry page) + * 1 PT (per-cpu page table page) + * ============ For the idle loop ============= + * 2 UPAGEs (per-cpu idle procs) + * 1 PTD (for per-cpu equiv of IdlePTD) + * ============================================ + * = total of 6 pages per cpu. The BSP reuses the ones allocated + * by locore.s during boot to remove special cases at runtime. + */ + ppage_map = kmem_suballoc(kernel_map, &minaddr, &maxaddr, + (NCPU*6*PAGE_SIZE), FALSE); +#endif + /* * Finally, allocate mbuf pool. Since mclrefcnt is an off-size * we use the more space efficient malloc in place of kmem_alloc. @@ -748,10 +772,24 @@ SYSCTL_INT(_machdep, CPU_WALLCLOCK, wall_cmos_clock, int currentldt; int _default_ldt; +#ifdef SMP +union descriptor gdt[NGDT + NCPU]; /* global descriptor table */ +#else union descriptor gdt[NGDT]; /* global descriptor table */ +#endif struct gate_descriptor idt[NIDT]; /* interrupt descriptor table */ union descriptor ldt[NLDT]; /* local descriptor table */ +#ifdef SMP +/* table descriptors - used to load tables by microp */ +struct region_descriptor r_gdt, r_idt; +#endif + +#ifdef SMP +struct i386tss SMPcommon_tss[NCPU]; /* One tss per cpu */ +struct i386tss *SMPcommon_tss_ptr[NCPU]; /* for the benefit of asmp code */ +#else struct i386tss common_tss; +#endif static struct i386tss dblfault_tss; static char dblfault_stack[PAGE_SIZE]; @@ -764,7 +802,11 @@ int gsel_tss; #endif /* software prototypes -- in more palatable form */ -struct soft_segment_descriptor gdt_segs[] = { +struct soft_segment_descriptor gdt_segs[ +#ifdef SMP + NGDT + NCPU +#endif + ] = { /* GNULL_SEL 0 Null Descriptor */ { 0x0, /* segment base address */ 0x0, /* length */ @@ -820,7 +862,12 @@ struct soft_segment_descriptor gdt_segs[] = { 0, /* unused - default 32 vs 16 bit size */ 0 /* limit granularity (byte/page units)*/ }, /* GPROC0_SEL 6 Proc 0 Tss Descriptor */ -{ (int) &common_tss, /* segment base address */ +{ +#ifdef SMP + (int) &SMPcommon_tss[0],/* segment base address */ +#else + (int) &common_tss, /* segment base address */ +#endif sizeof(struct i386tss)-1,/* length - all address space */ SDT_SYS386TSS, /* segment type */ 0, /* segment descriptor priority level */ @@ -968,8 +1015,10 @@ init386(first) int gsel_tss; #endif struct isa_device *idp; +#ifndef SMP /* table descriptors - used to load tables by microp */ struct region_descriptor r_gdt, r_idt; +#endif int pagesinbase, pagesinext; int target_page, pa_indx; int off; @@ -1002,6 +1051,18 @@ init386(first) for (x = 0; x < NGDT; x++) ssdtosd(&gdt_segs[x], &gdt[x].sd); +#ifdef SMP + /* + * Oh puke! + */ + for (x = 0; x < NCPU; x++) { + SMPcommon_tss_ptr[x] = &SMPcommon_tss[x]; + gdt_segs[NGDT + x] = gdt_segs[GPROC0_SEL]; + gdt_segs[NGDT + x].ssd_base = (int) SMPcommon_tss_ptr[x]; + ssdtosd(&gdt_segs[NGDT + x], &gdt[NGDT + x].sd); + } +#endif + /* make ldt memory segments */ /* * The data segment limit must not cover the user area because we @@ -1152,7 +1213,13 @@ init386(first) bootinfo.bi_extmem, biosextmem); } +#ifdef SMP + /* make hole for AP bootstrap code */ + pagesinbase = mp_bootaddress(biosbasemem) / PAGE_SIZE; +#else pagesinbase = biosbasemem * 1024 / PAGE_SIZE; +#endif + pagesinext = biosextmem * 1024 / PAGE_SIZE; /* @@ -1189,6 +1256,11 @@ init386(first) /* call pmap initialization to make new kernel address space */ pmap_bootstrap (first, 0); +#ifdef SMP + /* fire up the APs and APICs */ + mp_start(); +#endif + /* * Size up each available chunk of physical memory. */ @@ -1315,12 +1387,23 @@ init386(first) avail_end + off, VM_PROT_ALL, TRUE); msgbufmapped = 1; +#ifdef SMP + for(x = 0; x < NCPU; x++) { + /* make an initial tss so cpu can get interrupt stack on syscall! */ + SMPcommon_tss[x].tss_esp0 = (int) proc0.p_addr + UPAGES*PAGE_SIZE; + SMPcommon_tss[x].tss_ss0 = GSEL(GDATA_SEL, SEL_KPL) ; + SMPcommon_tss[x].tss_ioopt = (sizeof SMPcommon_tss[x]) << 16; + } + gsel_tss = GSEL(NGDT + cpunumber(), SEL_KPL); + ltr(gsel_tss); +#else /* make an initial tss so cpu can get interrupt stack on syscall! */ common_tss.tss_esp0 = (int) proc0.p_addr + UPAGES*PAGE_SIZE; common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL) ; common_tss.tss_ioopt = (sizeof common_tss) << 16; gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); ltr(gsel_tss); +#endif dblfault_tss.tss_esp = dblfault_tss.tss_esp0 = dblfault_tss.tss_esp1 = dblfault_tss.tss_esp2 = (int) &dblfault_stack[sizeof(dblfault_stack)]; @@ -1361,6 +1444,7 @@ init386(first) /* setup proc 0's pcb */ proc0.p_addr->u_pcb.pcb_flags = 0; proc0.p_addr->u_pcb.pcb_cr3 = IdlePTD; + proc0.p_addr->u_pcb.pcb_mpnest = 1; } /* diff --git a/sys/i386/i386/microtime.s b/sys/i386/i386/microtime.s index 6da8b78ed5be..6123e41fb659 100644 --- a/sys/i386/i386/microtime.s +++ b/sys/i386/i386/microtime.s @@ -32,12 +32,16 @@ * SUCH DAMAGE. * * from: Steve McCanne's microtime code - * $Id$ + * $Id: microtime.s,v 1.20 1997/02/22 09:32:34 peter Exp $ */ +#include "opt_smp.h" #include "opt_cpu.h" #include <machine/asmacros.h> +#ifdef APIC_IO +#include <machine/apic.h> +#endif /* APIC_IO */ #include <i386/isa/icu.h> #include <i386/isa/isa.h> @@ -45,7 +49,7 @@ ENTRY(microtime) -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) & !defined(SMP) movl _i586_ctr_freq, %ecx testl %ecx, %ecx jne pentium_microtime @@ -109,15 +113,26 @@ ENTRY(microtime) movl _timer0_max_count, %edx /* prepare for 2 uses */ +#if defined(APIC_IO) + movl _ipending, %eax + testl %eax, _mask8254 /* is soft timer interrupt pending? */ +#else testb $IRQ0, _ipending /* is a soft timer interrupt pending? */ +#endif /* APIC_IO */ jne overflow /* Do we have a possible overflow condition? */ cmpl _timer0_overflow_threshold, %ecx jbe 1f +#if defined(APIC_IO) + movl _apic_base, %eax + movl APIC_IRR1(%eax), %eax /** XXX assumption: IRQ0-24 */ + testl %eax, _mask8254 /* is a hard timer interrupt pending? */ +#else inb $IO_ICU1, %al /* read IRR in ICU */ testb $IRQ0, %al /* is a hard timer interrupt pending? */ +#endif /* APIC_IO */ je 1f overflow: subl %edx, %ecx /* some intr pending, count timer down through 0 */ @@ -183,7 +198,7 @@ common_microtime: ret -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) ALIGN_TEXT pentium_microtime: pushfl diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c new file mode 100644 index 000000000000..07fca34efe56 --- /dev/null +++ b/sys/i386/i386/mp_machdep.c @@ -0,0 +1,1646 @@ +/* + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: mp_machdep.c,v 1.47 1997/04/26 08:11:49 peter Exp $ + */ + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" +#if defined(APIC_IO) +#if !defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#else /* APIC_IO */ +#if defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#endif /* APIC_IO */ +#define FIX_MP_TABLE_WORKS_NOT + +#include "opt_serial.h" + +#include <sys/param.h> /* for KERNBASE */ +#include <sys/types.h> +#include <sys/sysproto.h> +#include <sys/time.h> +#include <sys/systm.h> + +#include <vm/vm.h> /* for KERNBASE */ +#include <vm/vm_param.h> /* for KERNBASE */ +#include <vm/pmap.h> /* for KERNBASE */ +#include <machine/pmap.h> /* for KERNBASE */ + +#include <machine/smp.h> +#include <machine/apic.h> +#include <machine/mpapic.h> +#include <machine/cpufunc.h> +#include <machine/segments.h> +#include <machine/smptests.h> /** TEST_UPPERPRIO, TEST_DEFAULT_CONFIG */ + +#include <i386/i386/cons.h> /* cngetc() */ + +#if defined(IPI_INTS) +#include <i386/isa/isa_device.h> +#include "vector.h" +#endif /* IPI_INTS */ + +#if defined(SMP_INVLTLB) +#include <i386/isa/icu.h> +#endif /* SMP_INVLTLB */ + +#define WARMBOOT_TARGET 0 +#define WARMBOOT_OFF (KERNBASE + 0x0467) +#define WARMBOOT_SEG (KERNBASE + 0x0469) + +#define BIOS_BASE (0xf0000) +#define BIOS_SIZE (0x10000) +#define BIOS_COUNT (BIOS_SIZE/4) + +#define CMOS_REG (0x70) +#define CMOS_DATA (0x71) +#define BIOS_RESET (0x0f) +#define BIOS_WARM (0x0a) + +/* + * this code MUST be enabled here and in mpboot.s. + * it follows the very early stages of AP boot by placing values in CMOS ram. + * it NORMALLY will never be needed and thus the primitive method for enabling. + * +#define CHECK_POINTS + */ + +#if defined(CHECK_POINTS) +#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) +#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) + +#define CHECK_INIT(D); \ + CHECK_WRITE(0x34, (D)); \ + CHECK_WRITE(0x35, (D)); \ + CHECK_WRITE(0x36, (D)); \ + CHECK_WRITE(0x37, (D)); \ + CHECK_WRITE(0x38, (D)); \ + CHECK_WRITE(0x39, (D)); + +#define CHECK_PRINT(S); \ + printf("%s: %d, %d, %d, %d, %d, %d\n", \ + (S), \ + CHECK_READ(0x34), \ + CHECK_READ(0x35), \ + CHECK_READ(0x36), \ + CHECK_READ(0x37), \ + CHECK_READ(0x38), \ + CHECK_READ(0x39)); + +#else /* CHECK_POINTS */ + +#define CHECK_INIT(D) +#define CHECK_PRINT(S) + +#endif /* CHECK_POINTS */ + + +/** FIXME: what system files declare these??? */ +extern struct region_descriptor r_gdt, r_idt; + +/* global data */ +struct proc *SMPcurproc[NCPU]; +struct pcb *SMPcurpcb[NCPU]; +struct timeval SMPruntime[NCPU]; + +int mp_ncpus; /* # of CPUs, including BSP */ +int mp_naps; /* # of Applications processors */ +int mp_nbusses; /* # of busses */ +int mp_napics; /* # of IO APICs */ +int mpenabled; +int boot_cpu_id; /* designated BSP */ +vm_offset_t cpu_apic_address; +vm_offset_t io_apic_address[NAPIC]; + +u_int32_t cpu_apic_versions[NCPU]; +u_int32_t io_apic_versions[NAPIC]; + +/* + * APIC ID logical/physical mapping structures + */ +int cpu_num_to_apic_id[NCPU]; +int io_num_to_apic_id[NAPIC]; +int apic_id_to_logical[NAPICID]; + +/* + * look for MP compliant motherboard. + */ + +static u_int boot_address; +static u_int base_memory; + +static int picmode; /* 0: virtual wire mode, 1: PIC mode */ +static u_int mpfps; +static int search_for_sig(u_int32_t target, int count); +static int mp_probe(u_int base_top); +static void mp_enable(u_int boot_addr); + +#if defined(IPI_INTS) +static void ipi_intr0(void); +static void ipi_intr1(void); +static void ipi_intr2(void); +static void ipi_intr3(void); +static int +ipi_ihandler_attach(int irq, inthand2_t * func, + unsigned *maskptr, int unit); +#endif /* IPI_INTS */ + + +/* + * calculate usable address in base memory for AP trampoline code + */ +u_int +mp_bootaddress(u_int basemem) +{ + base_memory = basemem * 1024; /* convert to bytes */ + + boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ + if ((base_memory - boot_address) < bootMP_size) + boot_address -= 4096; /* not enough, lower by 4k */ + + return boot_address; +} + + +/* + * startup the SMP processors + */ +void +mp_start(void) +{ + /* look for MP capable motherboard */ + if (mp_probe(base_memory)) + mp_enable(boot_address); + else { + printf("MP FPS NOT FOUND, suggest use of 'mptable' program\n"); + panic("can't continue!\n"); + } + + /* finish pmap initialization - turn off V==P mapping at zero */ + pmap_bootstrap2(); +} + + +/* + * print various information about the SMP system hardware and setup + */ +void +mp_announce(void) +{ + int x; + + printf("FreeBSD/SMP: Multiprocessor motherboard\n"); + printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); + printf(", version: 0x%08x\n", cpu_apic_versions[0]); + for (x = 1; x <= mp_naps; ++x) { + printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); + printf(", version: 0x%08x\n", cpu_apic_versions[x]); + } + +#if defined(APIC_IO) + for (x = 0; x < mp_napics; ++x) { + printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); + printf(", version: 0x%08x\n", io_apic_versions[x]); + } +#else + printf(" Warning: APIC I/O disabled\n"); +#endif /* APIC_IO */ +} + + +/* + * AP cpu's call this to sync up protected mode. + */ +void +init_secondary(void) +{ + int gsel_tss, slot; + + r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; + r_gdt.rd_base = (int) gdt; + lgdt(&r_gdt); /* does magic intra-segment return */ + lidt(&r_idt); + lldt(_default_ldt); + + slot = NGDT + cpunumber(); + gsel_tss = GSEL(slot, SEL_KPL); + gdt[slot].sd.sd_type = SDT_SYS386TSS; + ltr(gsel_tss); + + load_cr0(0x8005003b); /* XXX! */ +} + + +#if defined(APIC_IO) +void +configure_local_apic(void) +{ + u_char byte; + u_int32_t temp; + + if (picmode) { + outb(0x22, 0x70); /* select IMCR */ + byte = inb(0x23); /* current contents */ + byte |= 0x01; /* mask external INTR */ + outb(0x23, byte); /* disconnect 8259s/NMI */ + } + /* mask the LVT1 */ + temp = apic_base[APIC_LVT1]; + temp |= APIC_LVT_M; + apic_base[APIC_LVT1] = temp; +} +#endif /* APIC_IO */ + + +/******************************************************************* + * local functions and data + */ + +static int +mp_probe(u_int base_top) +{ + int x; + u_long segment; + u_int32_t target; + + /* see if EBDA exists */ + if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { + /* search first 1K of EBDA */ + target = (u_int32_t) (segment << 4); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } else { + /*last 1K of base memory, effective 'top of base' is passed in*/ + target = (u_int32_t) (base_top - 0x400); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } + + /* search the BIOS */ + target = (u_int32_t) BIOS_BASE; + if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) + goto found; + + /* nothing found */ + mpfps = mpenabled = 0; + return 0; + +found: /* please forgive the 'goto'! */ + /* flag fact that we are running multiple processors */ + mpfps = x; + mpenabled = 1; + return 1; +} + + +/* + * start the SMP system + */ +static int parse_mp_table(void); +static void default_mp_table(int type); +static int start_all_aps(u_int boot_addr); + +#if defined(XFAST_IPI32) +#include <machine/md_var.h> +#include <i386/isa/isa_device.h> +extern void Xfastipi32(u_int, u_int, u_int, u_int); +#endif /* XFAST_IPI32 */ + +static void +mp_enable(u_int boot_addr) +{ + int x; +#if defined(APIC_IO) + int apic; + u_int ux; +#if defined(TEST_UPPERPRIO) + u_char select; /* the select register is 8 bits */ + u_int32_t flags; /* the window register is 32 bits */ +#endif /* TEST_UPPERPRIO */ +#endif /* APIC_IO */ + + /* examine the MP table for needed info */ + x = parse_mp_table(); + + /* create pages for (address common) cpu APIC and each IO APIC */ + pmap_bootstrap_apics(); + + /* can't process default configs till the CPU APIC is pmapped */ + if (x) + default_mp_table(x); + +#if defined(APIC_IO) + /* fill the LOGICAL io_apic_versions table */ + for (apic = 0; apic < mp_napics; ++apic) { + ux = io_apic_read(apic, IOAPIC_VER); + io_apic_versions[apic] = ux; + } + + /* + */ + for (apic = 0; apic < mp_napics; ++apic) + if (io_apic_setup(apic) < 0) + panic("IO APIC setup failure\n"); + +#if defined(IPI_INTS) + /* setup IPI INTerrupt mechanism */ + ipi_ihandler_attach( /* irq */ 24, + /* XXX */ (inthand2_t *) ipi_intr0, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 25, + /* XXX */ (inthand2_t *) ipi_intr1, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 26, + /* XXX */ (inthand2_t *) ipi_intr2, NULL, /* unit */ 0); +#if !defined(SMP_INVLTLB) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); +#else +#if defined(XFAST_IPI32) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); + setidt(ICU_OFFSET + 32, + Xfastipi32, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#else + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_invltlb, NULL, /* unit */ 0); +#endif /* XFAST_IPI32 */ +#endif + +#endif /* IPI_INTS */ + +#if defined(TEST_UPPERPRIO) + +#if 1 + printf("special IRQ10\n"); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~0xff; /** clear vector */ + flags |= 64; + io_apic_write(0, select, flags); +#else + printf("special IRQ10\n"); + cngetc(); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~IOART_DELMOD; /* FIXED mode */ + io_apic_write(0, select, flags); + io_apic_write(0, select + 1, boot_cpu_id << 24); +#endif /** 0/1 */ + +#endif /* TEST_UPPERPRIO */ + +#endif /* APIC_IO */ + + /* start each Application Processor */ + start_all_aps(boot_addr); +} + + +/* + * look for the MP spec signature + */ + +/* string defined by the Intel MP Spec as identifying the MP table */ +#define MP_SIG 0x5f504d5f /* _MP_ */ +#define NEXT(X) ((X) += 4) +static int +search_for_sig(u_int32_t target, int count) +{ + int x; + u_int32_t *addr = (u_int32_t *) (KERNBASE + target); + + for (x = 0; x < count; NEXT(x)) + if (addr[x] == MP_SIG) + /* make array index a byte index */ + return (target + (x * sizeof(u_int32_t))); + + return -1; +} + + +#define PROCENTRY_FLAG_EN 0x01 +#define PROCENTRY_FLAG_BP 0x02 +#define IOAPICENTRY_FLAG_EN 0x01 + +/* MP Floating Pointer Structure */ +typedef struct MPFPS { + char signature[4]; + void *pap; + u_char length; + u_char spec_rev; + u_char checksum; + u_char mpfb1; + u_char mpfb2; + u_char mpfb3; + u_char mpfb4; + u_char mpfb5; +} *mpfps_t; +/* MP Configuration Table Header */ +typedef struct MPCTH { + char signature[4]; + u_short base_table_length; + u_char spec_rev; + u_char checksum; + u_char oem_id[8]; + u_char product_id[12]; + void *oem_table_pointer; + u_short oem_table_size; + u_short entry_count; + void *apic_address; + u_short extended_table_length; + u_char extended_table_checksum; + u_char reserved; +} *mpcth_t; + + +typedef struct PROCENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char cpu_flags; + u_long cpu_signature; + u_long feature_flags; + u_long reserved1; + u_long reserved2; +} *proc_entry_ptr; + +typedef struct BUSENTRY { + u_char type; + u_char bus_id; + char bus_type[6]; +} *bus_entry_ptr; + +typedef struct IOAPICENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char apic_flags; + void *apic_address; +} *io_apic_entry_ptr; + +typedef struct INTENTRY { + u_char type; + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} *int_entry_ptr; +/* descriptions of MP basetable entries */ +typedef struct BASETABLE_ENTRY { + u_char type; + u_char length; + char name[16]; +} basetable_entry; + +static basetable_entry basetable_entry_types[] = +{ + {0, 20, "Processor"}, + {1, 8, "Bus"}, + {2, 8, "I/O APIC"}, + {3, 8, "I/O INT"}, + {4, 8, "Local INT"} +}; + +typedef struct BUSDATA { + u_char bus_id; + enum busTypes bus_type; +} bus_datum; + +typedef struct INTDATA { + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} io_int, local_int; + +typedef struct BUSTYPENAME { + u_char type; + char name[7]; +} bus_type_name; + +static bus_type_name bus_type_table[] = +{ + {CBUS, "CBUS"}, + {CBUSII, "CBUSII"}, + {EISA, "EISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {ISA, "ISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {PCI, "PCI"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {XPRESS, "XPRESS"}, + {UNKNOWN_BUSTYPE, "---"} +}; +/* from MP spec v1.4, table 5-1 */ +static int default_data[7][5] = +{ +/* nbus, id0, type0, id1, type1 */ + {1, 0, ISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {0, 255, 255, 255, 255},/* MCA not supported */ + {2, 0, ISA, 1, PCI}, + {2, 0, EISA, 1, PCI}, + {0, 255, 255, 255, 255} /* MCA not supported */ +}; + + +/* the bus data */ +bus_datum bus_data[NBUS]; + +/* the IO INT data, one entry per possible APIC INTerrupt */ +io_int io_apic_ints[NINTR]; + +static int nintrs; + +#if defined(FIX_MP_TABLE_WORKS) +static void fix_mp_table __P((void)); +#endif /* FIX_MP_TABLE_WORKS */ + +static void processor_entry __P((proc_entry_ptr entry, int *cpu)); +static void io_apic_entry __P((io_apic_entry_ptr entry, int *apic)); +static void bus_entry __P((bus_entry_ptr entry, int *bus)); +static void int_entry __P((int_entry_ptr entry, int *intr)); +static int lookup_bus_type __P((char *name)); + + +/* + * parse an Intel MP specification table + */ +static int +parse_mp_table(void) +{ + int x; + mpfps_t fps; + mpcth_t cth; + int totalSize; + void *position; + int count; + int type; + int apic, bus, cpu, intr; + + /* clear physical APIC ID to logical CPU/IO table */ + for (x = 0; x < NAPICID; ++x) + ID_TO_IO(x) = -1; + + /* clear logical CPU to APIC ID table */ + for (x = 0; x < NCPU; ++x) + CPU_TO_ID(x) = -1; + + /* clear logical IO to APIC ID table */ + for (x = 0; x < NAPIC; ++x) + IO_TO_ID(x) = -1; + + /* clear IO APIC address table */ + for (x = 0; x < NAPIC; ++x) + io_apic_address[x] = ~0; + + /* clear bus data table */ + for (x = 0; x < NBUS; ++x) + bus_data[x].bus_id = 0xff; + + /* clear IO APIC INT table */ + for (x = 0; x < NINTR; ++x) + io_apic_ints[x].int_type = 0xff; + nintrs = 0; + + /* count the BSP */ + mp_ncpus = 1; + + /* setup the cpu/apic mapping arrays */ + boot_cpu_id = -1; + + /* local pointer */ + fps = (mpfps_t) mpfps; + + /* record whether PIC or virtual-wire mode */ + picmode = (fps->mpfb2 & 0x80) ? 1 : 0; + + /* check for use of 'default' configuration */ +#if defined(TEST_DEFAULT_CONFIG) + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return TEST_DEFAULT_CONFIG; +#else + if (fps->mpfb1 != 0) { + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return fps->mpfb1; + } +#endif /* TEST_DEFAULT_CONFIG */ + + if ((cth = fps->pap) == 0) + panic("MP Configuration Table Header MISSING!\n"); + + cpu_apic_address = (vm_offset_t) cth->apic_address; + + totalSize = cth->base_table_length - sizeof(struct MPCTH); + position = (u_char *) cth + sizeof(struct MPCTH); + count = cth->entry_count; + + apic = 0; /* logical apic# start @ 0 */ + bus = 0; /* logical bus# start @ 0 */ + cpu = 1; /* logical cpu# start @ 0, BUT reserve 0 for */ + /* BSP */ + intr = 0; /* unknown */ + + /* walk the table, recording info of interest */ + while (count--) { + switch (type = *(u_char *) position) { + case 0: + processor_entry(position, &cpu); + break; + case 1: + bus_entry(position, &bus); + break; + case 2: + io_apic_entry(position, &apic); + break; + case 3: + int_entry(position, &intr); + break; + case 4: + /* int_entry(position); */ + break; + default: + panic("mpfps Base Table HOSED!\n"); + /* NOTREACHED */ + } + + totalSize -= basetable_entry_types[type].length; + (u_char *) position += basetable_entry_types[type].length; + } + + if (boot_cpu_id == -1) + panic("NO BSP found!\n"); + + /* record # of APs found */ + mp_naps = (cpu - 1); + + /* record # of busses found */ + mp_nbusses = bus; + + /* record # of IO APICs found */ + mp_napics = apic; + + /* record # of IO APICs found */ + nintrs = intr; + +#if defined(FIX_MP_TABLE_WORKS) + /* post scan cleanup */ + fix_mp_table(); +#endif /* FIX_MP_TABLE_WORKS */ + + /* report fact that its NOT a default configuration */ + return 0; +} + + +/* + * parse an Intel MP specification table + */ +#if defined(FIX_MP_TABLE_WORKS) +static void +fix_mp_table(void) +{ + int x; + int y; + int num_pci_bus; + bus_datum bus_record; + + /* + * Fix mis-numbering of the PCI bus and its INT entries if the BIOS + * did it wrong. The MP spec says that when more than 1 PCI bus + * exists the BIOS must begin with bus entries for the PCI bus and use + * actual PCI bus numbering. This implies that when only 1 PCI bus + * exists the BIOS can choose to ignore this ordering, and indeed many + * MP motherboards do ignore it. This causes a problem when the PCI + * sub-system makes requests of the MP sub-system based on PCI bus + * numbers. So here we look for the situation and renumber the + * busses and associated INTs in an effort to "make it right". + */ + + /* count the number of PCI busses */ + for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + ++num_pci_bus; + } + + /* check the 1 PCI bus case for sanity */ + if (num_pci_bus == 1) { + + /* if its in the first slot all is well */ + if (bus_data[0].bus_type == PCI) + return; + + /* mis-numbered, swap with whichever bus uses slot 0 */ + + /* locate the entry holding the PCI bus */ + for (x = 1; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + break; + } + + /* swap the bus entry records */ + bus_record = bus_data[0]; + bus_data[0] = bus_data[x]; + bus_data[x] = bus_record; + + /* swap each relavant INTerrupt entry */ + for (y = 0; y < nintrs; ++y) { + if (io_apic_ints[y].src_bus_id == x) + io_apic_ints[y].src_bus_id = 0; + else + if (io_apic_ints[y].src_bus_id == 0) + io_apic_ints[y].src_bus_id = x; + } + } + /* sanity check if more than 1 PCI bus */ + else + if (num_pci_bus > 1) { + for (x = 0; x < num_pci_bus; ++x) { + if (bus_data[x].bus_type != PCI) { + printf("bad PCI bus numbering\n"); + panic("\n"); + } + } + } +} +#endif /* FIX_MP_TABLE_WORKS */ + + +static void +processor_entry(proc_entry_ptr entry, int *cpu) +{ + int x = *cpu; + + /* check for usability */ + if (!(entry->cpu_flags & PROCENTRY_FLAG_EN)) + return; + + /* check for BSP flag */ + if (entry->cpu_flags & PROCENTRY_FLAG_BP) { + /* always give boot CPU the logical value of 0 */ + x = 0; + boot_cpu_id = entry->apic_id; + } else { + /* add another AP to list, if less than max number of CPUs */ + if (x == NCPU) { + printf("Warning: only using %d of the available CPUs!\n", x); + return; + } + ++(*cpu); + } + + CPU_TO_ID(x) = entry->apic_id; + ID_TO_CPU(entry->apic_id) = x; +} + + +static void +bus_entry(bus_entry_ptr entry, int *bus) +{ + int x, y; + char name[8]; + char c; + + if ((x = (*bus)++) == NBUS) + panic("too many busses, increase 'NBUS'\n"); + + /* encode the name into an index */ + for (y = 0; y < 6; ++y) { + if ((c = entry->bus_type[y]) == ' ') + break; + name[y] = c; + } + name[y] = '\0'; + + if ((y = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) + panic("unknown bus type: '%s'\n", name); + + bus_data[x].bus_id = entry->bus_id; + bus_data[x].bus_type = y; +} + + +static void +io_apic_entry(io_apic_entry_ptr entry, int *apic) +{ + int x; + + if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) + return; + + if ((x = (*apic)++) == NAPIC) + panic("too many APICs, increase 'NAPIC'\n"); + + IO_TO_ID(x) = entry->apic_id; + ID_TO_IO(entry->apic_id) = x; + + io_apic_address[x] = (vm_offset_t) entry->apic_address; +} + + +static int +lookup_bus_type(char *name) +{ + int x; + + for (x = 0; x < MAX_BUSTYPE; ++x) + if (strcmp(bus_type_table[x].name, name) == 0) + return bus_type_table[x].type; + + return UNKNOWN_BUSTYPE; +} + + +static void +int_entry(int_entry_ptr entry, int *intr) +{ + int x; + + if ((x = (*intr)++) == NINTR) + panic("too many INTs, increase 'NINTR'\n"); + + io_apic_ints[x].int_type = entry->int_type; + io_apic_ints[x].int_flags = entry->int_flags; + io_apic_ints[x].src_bus_id = entry->src_bus_id; + io_apic_ints[x].src_bus_irq = entry->src_bus_irq; + io_apic_ints[x].dst_apic_id = entry->dst_apic_id; + io_apic_ints[x].dst_apic_int = entry->dst_apic_int; +} + + +static int +apic_int_is_bus_type(int intr, int bus_type) +{ + int bus; + + for (bus = 0; bus < mp_nbusses; ++bus) + if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) + && ((int) bus_data[bus].bus_type == bus_type)) + return 1; + + return 0; +} + + +/* + * determine which APIC pin an ISA INT is attached to. + */ +#define INTTYPE(I) (io_apic_ints[(I)].int_type) +#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) + +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_isa_apic_irq(int isaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (isaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == isaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, ISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin an EISA INT is attached to. + */ +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_eisa_apic_irq(int eisaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (eisaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == eisaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, EISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin a PCI INT is attached to. + */ +#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) +#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) +#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) +int +get_pci_apic_irq(int pciBus, int pciDevice, int pciInt) +{ + int intr; + + --pciInt; /* zero based */ + + for (intr = 0; intr < nintrs; ++intr) /* search each record */ + if ((INTTYPE(intr) == 0) +#if defined(FIX_MP_TABLE_WORKS) + && (SRCBUSID(intr) == pciBus) +#endif /* FIX_MP_TABLE_WORKS */ + && (SRCBUSDEVICE(intr) == pciDevice) + && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, PCI)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSLINE +#undef SRCBUSDEVICE +#undef SRCBUSID + +#undef INTPIN +#undef INTTYPE + + +int +undirect_pci_irq(int rirq) +{ +#if defined(READY) + printf("Freeing irq %d for ISA cards.\n", rirq); + /** FIXME: tickle the MB redirector chip */ + return ???; +#else + printf("Freeing (NOT implimented) irq %d for ISA cards.\n", rirq); + return 0; +#endif /* READY */ +} + + +/* + * given a bus ID, return: + * the bus type if found + * -1 if NOT found + */ +int +apic_bus_type(int id) +{ + int x; + + for (x = 0; x < mp_nbusses; ++x) + if (bus_data[x].bus_id == id) + return bus_data[x].bus_type; + + return -1; +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus ID if found + * -1 if NOT found + */ +int +apic_src_bus_id(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_id); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus IRQ if found + * -1 if NOT found + */ +int +apic_src_bus_irq(int apic, int pin) +{ + int x; + + for (x = 0; x < nintrs; x++) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_irq); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated INTerrupt type if found + * -1 if NOT found + */ +int +apic_int_type(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_type); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated trigger mode if found + * -1 if NOT found + */ +int +apic_trigger(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return ((io_apic_ints[x].int_flags >> 2) & 0x03); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated 'active' level if found + * -1 if NOT found + */ +int +apic_polarity(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_flags & 0x03); + + return -1; /* NOT found */ +} + + +/* + * set data according to MP defaults + * FIXME: probably not complete yet... + */ +static void +default_mp_table(int type) +{ + int ap_cpu_id; +#if defined(APIC_IO) + u_int32_t ux; + int io_apic_id; + int pin; +#endif /* APIC_IO */ + +#if 0 + printf(" MP default config type: %d\n", type); + switch (type) { + case 1: + printf(" bus: ISA, APIC: 82489DX\n"); + break; + case 2: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 3: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 4: + printf(" bus: MCA, APIC: 82489DX\n"); + break; + case 5: + printf(" bus: ISA+PCI, APIC: Integrated\n"); + break; + case 6: + printf(" bus: EISA+PCI, APIC: Integrated\n"); + break; + case 7: + printf(" bus: MCA+PCI, APIC: Integrated\n"); + break; + default: + printf(" future type\n"); + break; + /* NOTREACHED */ + } +#endif /* 0 */ + + boot_cpu_id = (apic_base[APIC_ID] & APIC_ID_MASK) >> 24; + ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; + + /* BSP */ + CPU_TO_ID(0) = boot_cpu_id; + ID_TO_CPU(boot_cpu_id) = 0; + + /* one and only AP */ + CPU_TO_ID(1) = ap_cpu_id; + ID_TO_CPU(ap_cpu_id) = 1; + mp_naps = 1; + + /* one and only IO APIC */ +#if defined(APIC_IO) + io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; + + /* + * sanity check, refer to MP spec section 3.6.6, last paragraph + * necessary as some hardware isn't properly setting up the IO APIC + */ +#if defined(REALLY_ANAL_IOAPICID_VALUE) + if (io_apic_id != 2) { +#else + if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { +#endif /* REALLY_ANAL_IOAPICID_VALUE */ + ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ + ux &= ~APIC_ID_MASK; /* clear the ID field */ + ux |= 0x02000000; /* set it to '2' */ + io_apic_write(0, IOAPIC_ID, ux); /* write new value */ + ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ + if ((ux & APIC_ID_MASK) != 0x02000000) + panic("Problem: can't control IO APIC ID, reg: 0x%08x\n", ux); + io_apic_id = 2; + } + IO_TO_ID(0) = io_apic_id; + ID_TO_IO(io_apic_id) = 0; + mp_napics = 1; +#else + mp_napics = 0; +#endif /* APIC_IO */ + + /* fill out bus entries */ + switch (type) { + case 1: + case 2: + case 3: + case 5: + case 6: + mp_nbusses = default_data[type - 1][0]; + bus_data[0].bus_id = default_data[type - 1][1]; + bus_data[0].bus_type = default_data[type - 1][2]; + bus_data[1].bus_id = default_data[type - 1][3]; + bus_data[1].bus_type = default_data[type - 1][4]; + break; + + /* case 4: case 7: MCA NOT supported */ + default: /* illegal/reserved */ + panic("BAD default MP config: %d\n", type); + } + +#if defined(APIC_IO) + /* general cases from MP v1.4, table 5-2 */ + for (pin = 0; pin < 16; ++pin) { + io_apic_ints[pin].int_type = 0; + io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ + io_apic_ints[pin].src_bus_id = 0; + io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ + io_apic_ints[pin].dst_apic_id = io_apic_id; + io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ + } + + /* special cases from MP v1.4, table 5-2 */ + if (type == 2) { + io_apic_ints[2].int_type = 0xff; /* N/C */ + io_apic_ints[13].int_type = 0xff; /* N/C */ +#if !defined(APIC_MIXED_MODE) + /** FIXME: ??? */ + panic("sorry, can't support type 2 default yet\n"); +#endif /* APIC_MIXED_MODE */ + } else + io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ + + if (type == 7) + io_apic_ints[0].int_type = 0xff; /* N/C */ + else + io_apic_ints[0].int_type = 3; /* vectored 8259 */ + + nintrs = 16; +#endif /* APIC_IO */ +} + + +static void install_ap_tramp(u_int boot_addr); +static int start_ap(int logicalCpu, u_int boot_addr); + +/* + * start each AP in our list + */ +static int +start_all_aps(u_int boot_addr) +{ + int x; + u_char mpbiosreason; + u_long mpbioswarmvec; + + /** + * NOTE: this needs further thought: + * where does it get released? + * should it be set to empy? + * + * get the initial mp_lock with a count of 1 for the BSP + */ + mp_lock = (apic_base[APIC_ID] & APIC_ID_MASK) + 1; + + /* initialize BSP's local APIC */ + apic_initialize(1); + + /* install the AP 1st level boot code */ + install_ap_tramp(boot_addr); + + /* save the current value of the warm-start vector */ + mpbioswarmvec = *((u_long *) WARMBOOT_OFF); + outb(CMOS_REG, BIOS_RESET); + mpbiosreason = inb(CMOS_DATA); + + /* start each AP */ + for (x = 1; x <= mp_naps; ++x) { + + /* setup a vector to our boot code */ + *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; + *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ + + /* attempt to start the Application Processor */ + CHECK_INIT(99); /* setup checkpoints */ + if (!start_ap(x, boot_addr)) { + printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); + CHECK_PRINT("trace"); /* show checkpoints */ + /* + * better panic as the AP may be running loose + * somewhere + */ + printf("panic y/n? [n] "); + if (cngetc() != 'n') + panic("bye-bye\n"); + } + CHECK_PRINT("trace"); /* show checkpoints */ + + /* record its version info */ + cpu_apic_versions[x] = cpu_apic_versions[0]; + } + + /* fill in our (BSP) APIC version */ + cpu_apic_versions[0] = apic_base[APIC_VER]; + + /* restore the warmstart vector */ + *(u_long *) WARMBOOT_OFF = mpbioswarmvec; + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, mpbiosreason); + + /* number of APs actually started */ + return mp_ncpus - 1; +} + + +/* + * load the 1st level AP boot code into base memory. + */ + +/* targets for relocation */ +extern void bigJump(void); +extern void bootCodeSeg(void); +extern void bootDataSeg(void); +extern void MPentry(void); +extern u_int MP_GDT; +extern u_int mp_gdtbase; + +static void +install_ap_tramp(u_int boot_addr) +{ + int x; + int size = *(int *) ((u_long) & bootMP_size); + u_char *src = (u_char *) ((u_long) bootMP); + u_char *dst = (u_char *) boot_addr + KERNBASE; + u_int boot_base = (u_int) bootMP; + u_int8_t *dst8; + u_int16_t *dst16; + u_int32_t *dst32; + + for (x = 0; x < size; ++x) + *dst++ = *src++; + + /* + * modify addresses in code we just moved to basemem. unfortunately we + * need fairly detailed info about mpboot.s for this to work. changes + * to mpboot.s might require changes here. + */ + + /* boot code is located in KERNEL space */ + dst = (u_char *) boot_addr + KERNBASE; + + /* modify the lgdt arg */ + dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); + *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); + + /* modify the ljmp target for MPentry() */ + dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); + *dst32 = ((u_int) MPentry - KERNBASE); + + /* modify the target for boot code segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; + + /* modify the target for boot data segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; +} + + +/* + * this function starts the AP (application processor) identified + * by the APIC ID 'physicalCpu'. It does quite a "song and dance" + * to accomplish this. This is necessary because of the nuances + * of the different hardware we might encounter. It ain't pretty, + * but it seems to work. + */ +static int +start_ap(int logical_cpu, u_int boot_addr) +{ + int physical_cpu; + int vector; + int cpus; + u_long icr_lo, icr_hi; + + /* get the PHYSICAL APIC ID# */ + physical_cpu = CPU_TO_ID(logical_cpu); + + /* calculate the vector */ + vector = (boot_addr >> 12) & 0xff; + + /* used as a watchpoint to signal AP startup */ + cpus = mp_ncpus; + + /* + * first we do an INIT/RESET IPI this INIT IPI might be run, reseting + * and running the target CPU. OR this INIT IPI might be latched (P5 + * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be + * ignored. + */ + + /* setup the address for the target AP */ + icr_hi = apic_base[APIC_ICR_HI] & ~APIC_ID_MASK; + icr_hi |= (physical_cpu << 24); + apic_base[APIC_ICR_HI] = icr_hi; + + /* do an INIT IPI: assert RESET */ + icr_lo = apic_base[APIC_ICR_LOW] & 0xfff00000; + apic_base[APIC_ICR_LOW] = icr_lo | 0x0000c500; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* do an INIT IPI: deassert RESET */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00008500; + + /* wait for pending status end */ + u_sleep(10000); /* wait ~10mS */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* + * next we do a STARTUP IPI: the previous INIT IPI might still be + * latched, (P5 bug) this 1st STARTUP would then terminate + * immediately, and the previously started INIT IPI would continue. OR + * the previous INIT IPI has already run. and this STARTUP IPI will + * run. OR the previous INIT IPI was ignored. and this STARTUP IPI + * will run. + */ + + /* do a STARTUP IPI */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* + * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF + * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR + * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is + * recognized after hardware RESET or INIT IPI. + */ + + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* wait for it to start */ + set_apic_timer(5000000);/* == 5 seconds */ + while (read_apic_timer()) + if (mp_ncpus > cpus) + return 1; /* return SUCCESS */ + + return 0; /* return FAILURE */ +} + + +#if defined(IPI_INTS) + +static void +ipi_intr0(void) +{ + printf("IPI 0\n"); +} + + +static void +ipi_intr1(void) +{ + printf("IPI 1\n"); +} + + +static void +ipi_intr2(void) +{ + printf("IPI 2\n"); +} + + +static void +ipi_intr3(void) +{ + printf("IPI 3\n"); +} + +/*----------------------------------------------------------------------- +** +** Register an interupt handler for an IPI. +** (Stolen from the PCI<->ISA glue code) +** +**----------------------------------------------------------------------- +*/ + +static int +ipi_ihandler_attach(int irq, inthand2_t * func, unsigned *maskptr, int unit) +{ + char buf[16]; + char *cp; + int free_id, id, result; + + sprintf(buf, "ipi irq%d", irq); + for (cp = intrnames, free_id = 0, id = 0; id < NR_DEVICES; id++) { + if (strcmp(cp, buf) == 0) + break; + if (free_id <= 0 && strcmp(cp, "ipi irqnn") == 0) + free_id = id; + while (*cp++ != '\0'); + } + if (id == NR_DEVICES) { + id = free_id; + if (id == 0) { + /* + * All ipi irq counters are in use, perhaps because + * config is old so there aren't any. Abuse the clk0 + * counter. + */ + printf( + "ipi_ihandler_attach: counting ipi irq%d's as clk0 irqs\n", + irq); + } + } + result = register_intr( + irq, /* isa irq */ + id, /* device id */ + 0, /* flags? */ + func, /* handler */ + maskptr, /* mask pointer */ + unit); /* handler arg */ + + if (result) { + printf("WARNING: ipi_ihandler_attach: result=%d\n", result); + return (result); + }; + + return (0); +} +#endif /* IPI_INTS */ + + +#ifdef SMP_INVLTLB +/* + * Flush the TLB on all other CPU's + * + * XXX: Needs to handshake and wait for completion before proceding. + */ + +void +smp_invltlb() +{ + if (smp_active) { + if (invldebug & 2) +#if defined(XFAST_IPI32) + all_but_self_ipi(ICU_OFFSET + 32); +#else + all_but_self_ipi(ICU_OFFSET + 27); +#endif /* XFAST_IPI32 */ + } +} + +void +invlpg(u_int addr) +{ + __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); + smp_invltlb(); +} + +void +invltlb(void) +{ + u_long temp; + /* + * This should be implemented as load_cr3(rcr3()) when load_cr3() is + * inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); + smp_invltlb(); +} +/* + * Handles recieving an "IRQ 27", the invalidate tlb IPI.. + */ +void +ipi_invltlb(void) +{ + u_long temp; + + if (invldebug & 4) { + /* + * This should be implemented as load_cr3(rcr3()) when + * load_cr3() is inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) + :: "memory"); + + } +} +#endif /* SMP_INVLTLB */ diff --git a/sys/i386/i386/mpapic.c b/sys/i386/i386/mpapic.c new file mode 100644 index 000000000000..b17d3788f950 --- /dev/null +++ b/sys/i386/i386/mpapic.c @@ -0,0 +1,832 @@ +/* + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: mpapic.c,v 1.32 1997/04/26 05:58:01 peter Exp $ + */ + +#include <sys/types.h> +#include <sys/sysproto.h> +#include <sys/systm.h> + +#include <machine/smp.h> +#include <machine/apic.h> /** IPI_INTS */ +#include <machine/mpapic.h> +#include <machine/smptests.h> /** TEST_LOPRIO, TEST_IPI */ +#include <machine/cpufunc.h> +#include <machine/segments.h> + + +/* EISA Edge/Level trigger control registers */ +#define ELCR0 0x4d0 /* eisa irq 0-7 */ +#define ELCR1 0x4d1 /* eisa irq 8-15 */ + +/* + * pointers to pmapped apic hardware. + */ +volatile u_int *apic_base; + +#if defined(APIC_IO) +volatile u_int *io_apic_base; +#endif /* APIC_IO */ + +/* + * enable APIC, configure interrupts + */ +void +apic_initialize(int is_bsp) +{ + u_int temp; + + if (is_bsp) { + /* setup LVT1 as ExtINT */ + temp = apic_base[APIC_LVT1]; + temp &= 0xfffe58ff; + temp |= 0x00000700; + apic_base[APIC_LVT1] = temp; + } + /* setup LVT2 as NMI */ + temp = apic_base[APIC_LVT2]; + temp &= 0xfffe58ff; + temp |= 0xffff0400; + apic_base[APIC_LVT2] = temp; + + /* set the Task Priority Register as needed */ + temp = apic_base[APIC_TPR]; + temp &= ~APIC_TPR_PRIO; /* clear priority field */ + +#if defined(TEST_LOPRIO) + if (is_bsp) + temp |= 0x10; /* allow INT arbitration */ + else + temp |= 0xff; /* disallow INT arbitration */ +#endif /* TEST_LOPRIO */ + + apic_base[APIC_TPR] = temp; + + /* enable the beast */ + temp = apic_base[APIC_SVR]; + temp |= APIC_SVR_SWEN; /* software enable APIC */ + temp &= ~APIC_SVR_FOCUS;/* enable 'focus processor' */ +#if 0 + temp |= 0x20; /** FIXME: 2f == strayIRQ15 */ +#endif + apic_base[APIC_SVR] = temp; +} + + +#if !defined(FAST_IPI) + +/* + * send an IPI INTerrupt containing 'vector' to CPUs in 'target_map' + * 'target_map' is a bitfiled of length 14, + * APIC #0 == bit 0, ..., APIC #14 == bit 14 + * NOTE: these are LOGICAL APIC IDs + */ +int +selected_procs_ipi(int target_map, int vector) +{ + return selected_apic_ipi(target_map, vector, APIC_DELMODE_FIXED); +} + +/* + * send an IPI INTerrupt containing 'vector' to all CPUs, including myself + */ +int +all_procs_ipi(int vector) +{ + return apic_ipi(APIC_DEST_ALLISELF, vector, APIC_DELMODE_FIXED); +} + +/* + * send an IPI INTerrupt containing 'vector' to all CPUs EXCEPT myself + */ +int +all_but_self_ipi(int vector) +{ + return apic_ipi(APIC_DEST_ALLESELF, vector, APIC_DELMODE_FIXED); +} + +/* + * send an IPI INTerrupt containing 'vector' to myself + */ +int +self_ipi(int vector) +{ + return apic_ipi(APIC_DEST_SELF, vector, APIC_DELMODE_FIXED); +} + +/* + * send APIC IPI 'vector' to 'destType' via 'deliveryMode' + * + * destType is 1 of: APIC_DEST_SELF, APIC_DEST_ALLISELF, APIC_DEST_ALLESELF + * vector is any valid SYSTEM INT vector + * delivery_mode is 1 of: APIC_DELMODE_FIXED, APIC_DELMODE_LOWPRIO + */ +int +apic_ipi(int dest_type, int vector, int delivery_mode) +{ +#if defined(APICIPI_BANDAID) +#define MAX_SPIN1 10000000 +#define MAX_SPIN2 1000 + + u_long icr_lo; + int x; + + /* "lazy delivery", ie we only barf if they stack up on us... */ + for (x = MAX_SPIN1; x; --x) { + if ((apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) == 0) + break; + } + if (x == 0) { + printf("apic_ipi was stuck\n"); + panic("\n"); + } + /* build IRC_LOW */ + icr_lo = (apic_base[APIC_ICR_LOW] & APIC_RESV2_MASK) + | dest_type | delivery_mode | vector; + + /* write APIC ICR */ + apic_base[APIC_ICR_LOW] = icr_lo; + + /* wait for pending status end */ + for (x = MAX_SPIN2; x; --x) { + if ((apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) == 0) + break; + } + if (x == 0) + printf("apic_ipi might be stuck\n"); + + /** FIXME: return result */ + return 0; +#else /* APICIPI_BANDAID */ + u_long icr_lo; + + /* build IRC_LOW */ + icr_lo = (apic_base[APIC_ICR_LOW] & APIC_RESV2_MASK) + | dest_type | delivery_mode | vector; + + /* write APIC ICR */ + apic_base[APIC_ICR_LOW] = icr_lo; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /** FIXME: return result */ + return 0; +#endif /* APICIPI_BANDAID */ +} +#endif /* FAST_IPI */ + + +/* + * send APIC IPI 'vector' to 'target's via 'delivery_mode' + * + * target contains a bitfield with a bit set for selected APICs. + * vector is any valid SYSTEM INT vector + * delivery_mode is 1 of: APIC_DELMODE_FIXED, APIC_DELMODE_LOWPRIO + */ +int +selected_apic_ipi(u_int target, int vector, int delivery_mode) +{ + int x; + int status; + u_long icr_hi; + + if (target & ~0x7fff) + return -1; /* only 15 targets allowed */ + + for (status = 0, x = 0; x <= 14; ++x) + if (target & (1 << x)) { + /* write the destination field for the target AP */ + icr_hi = apic_base[APIC_ICR_HI] & ~APIC_ID_MASK; + icr_hi |= (CPU_TO_ID(x) << 24); + apic_base[APIC_ICR_HI] = icr_hi; + + /* send the IPI */ + if (apic_ipi(APIC_DEST_DESTFLD, vector, delivery_mode) == -1) + status |= (1 << x); + } + return status; +} + + +#if defined(APIC_IO) + +/* + * IO APIC code, + */ + +#define IOAPIC_ISA_INTS 16 +#define REDIRCNT_IOAPIC(A) \ + ((int)((io_apic_versions[(A)] & IOART_VER_MAXREDIR) >> MAXREDIRSHIFT) + 1) + +static int trigger __P((int apic, int pin, u_int32_t * flags)); +static void polarity __P((int apic, int pin, u_int32_t * flags, int level)); + + +/* + * setup I/O APIC + */ + +#ifdef HOW_8259_IS_PROGRAMMED +outb(IO_ICU1, 0x11); /* reset; program device, four bytes */ +outb(IO_ICU1 + 1, NRSVIDT); /* starting at this vector index */ +outb(IO_ICU1 + 1, 1 << 2); /* slave on line 2 */ +outb(IO_ICU1 + 1, 1); /* 8086 mode */ + +outb(IO_ICU1 + 1, 0xff); /* leave interrupts masked */ +outb(IO_ICU1, 0x0a); /* default to IRR on read */ +outb(IO_ICU1, 0xc0 | (3 - 1)); /* pri order 3-7, 0-2 (com2 first) */ + +outb(IO_ICU2, 0x11); /* reset; program device, four bytes */ +outb(IO_ICU2 + 1, NRSVIDT + 8); /* staring at this vector index */ +outb(IO_ICU2 + 1, 2); /* my slave id is 2 */ +outb(IO_ICU2 + 1, 1); /* 8086 mode */ + +outb(IO_ICU2 + 1, 0xff); /* leave interrupts masked */ +outb(IO_ICU2, 0x0a); /* default to IRR on read */ +#endif /* HOW_8259_IS_PROGRAMMED */ + + +#if defined(TEST_LOPRIO) +#define DEFAULT_FLAGS \ + ((u_int32_t) \ + (IOART_INTMSET | \ + IOART_DESTPHY | \ + IOART_DELLOPRI)) + +#define DEFAULT_ISA_FLAGS \ + ((u_int32_t) \ + (IOART_INTMSET | \ + IOART_TRGREDG | \ + IOART_INTAHI | \ + IOART_DESTPHY | \ + IOART_DELLOPRI)) +#else +#define DEFAULT_FLAGS \ + ((u_int32_t) \ + (IOART_INTMSET | \ + IOART_DESTPHY | \ + IOART_DELFIXED)) + +#define DEFAULT_ISA_FLAGS \ + ((u_int32_t) \ + (IOART_INTMSET | \ + IOART_TRGREDG | \ + IOART_INTAHI | \ + IOART_DESTPHY | \ + IOART_DELFIXED)) +#endif /* TEST_LOPRIO */ + +/* + * + */ +int +io_apic_setup(int apic) +{ + int maxpin; + u_char select; /* the select register is 8 bits */ + u_int32_t flags; /* the window register is 32 bits */ + u_int32_t target; /* the window register is 32 bits */ + u_int32_t vector; /* the window register is 32 bits */ + int pin, level; + +#if defined(TEST_LOPRIO) + target = IOART_DEST; +#else + target = boot_cpu_id << 24; +#endif /* TEST_LOPRIO */ + + if (apic == 0) { + /* APIC[0] INT0 thru INT15 default to ISA */ + select = IOAPIC_REDTBL0; + vector = NRSVIDT; + for (pin = 0; + pin < IOAPIC_ISA_INTS; + ++pin, ++vector, select += 2) { + io_apic_write(apic, select, DEFAULT_ISA_FLAGS | vector); + io_apic_write(apic, select + 1, target); + } + + /* remainder of APIC[0] */ + if ((maxpin = REDIRCNT_IOAPIC(apic)) > IOAPIC_ISA_INTS) { + select = IOAPIC_REDTBL16; + vector = NRSVIDT + IOAPIC_ISA_INTS; + for (pin = IOAPIC_ISA_INTS; + pin < maxpin; + ++pin, ++vector, select += 2) { + + if (apic_int_type(apic, pin) != 0) + continue; + + flags = DEFAULT_FLAGS; + level = trigger(apic, pin, &flags); + polarity(apic, pin, &flags, level); + + io_apic_write(apic, select, flags | vector); + io_apic_write(apic, select + 1, target); + } + } + } else { + + /* program entry according to MP table. */ +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX +#else + panic("ioApicSetup: apic #%d\n", apic); +#endif /* MULTIPLE_IOAPICS */ + + } + + /* return GOOD status */ + return 0; +} +#undef DEFAULT_ISA_FLAGS +#undef DEFAULT_FLAGS + + +#define DEFAULT_EXTINT_FLAGS \ + ((u_int32_t) \ + (IOART_INTMSET | \ + IOART_TRGREDG | \ + IOART_INTAHI | \ + IOART_DESTPHY | \ + IOART_DELEXINT)) + +/* + * + */ +int +ext_int_setup(int apic, int intr) +{ + u_char select; /* the select register is 8 bits */ + u_int32_t flags; /* the window register is 32 bits */ + u_int32_t target; /* the window register is 32 bits */ + u_int32_t vector; /* the window register is 32 bits */ + + if (apic_int_type(apic, intr) != 3) + return -1; + +/** FIXME: where should we steer ExtInts ??? */ +#if defined(BROADCAST_EXTINT) + target = IOART_DEST; +#else + target = boot_cpu_id << 24; +#endif /* BROADCAST_EXTINT */ + + select = IOAPIC_REDTBL0 + (2 * intr); + vector = NRSVIDT + intr; + flags = DEFAULT_EXTINT_FLAGS; + + io_apic_write(apic, select, flags | vector); + io_apic_write(apic, select + 1, target); + + return 0; +} +#undef DEFAULT_EXTINT_FLAGS + + +/* + * + */ +static int +trigger(int apic, int pin, u_int32_t * flags) +{ + int id; + int eirq; + int level; + static int intcontrol = -1; + + switch (apic_trigger(apic, pin)) { + + case 0x00: + break; + + case 0x01: + *flags &= ~IOART_TRGRLVL; /* *flags |= IOART_TRGREDG */ + return 0; + + case 0x03: + *flags |= IOART_TRGRLVL; + return 1; + + case -1: + default: + goto bad; + } + + if ((id = apic_src_bus_id(apic, pin)) == -1) + goto bad; + + switch (apic_bus_type(id)) { + case ISA: + *flags &= ~IOART_TRGRLVL; /* *flags |= IOART_TRGREDG; */ + return 0; + + case EISA: + eirq = apic_src_bus_irq(apic, pin); + if (eirq < 0 || eirq > 15) { + printf("EISA IRQ %d?!?!\n", eirq); + goto bad; + } + if (intcontrol == -1) { + intcontrol = inb(ELCR1) << 8; + intcontrol |= inb(ELCR0); + printf("EISA INTCONTROL = %08x\n", intcontrol); + } + /* EISA IRQ's are identical to ISA irq's, regardless of + * whether they are edge or level since they go through the + * level/polarity converter gadget. */ + level = 0; + + if (level) + *flags |= IOART_TRGRLVL; + else + *flags &= ~IOART_TRGRLVL; + + return level; + + case PCI: + *flags |= IOART_TRGRLVL; + return 1; + + case -1: + default: + goto bad; + } + +bad: + panic("bad APIC IO INT flags\n"); +} + + +/* + * + */ +static void +polarity(int apic, int pin, u_int32_t * flags, int level) +{ + int id; + int eirq; + int pol; + + switch (apic_polarity(apic, pin)) { + + case 0x00: + break; + + case 0x01: + *flags &= ~IOART_INTALO; /* *flags |= IOART_INTAHI */ + return; + + case 0x03: + *flags |= IOART_INTALO; + return; + + case -1: + default: + goto bad; + } + + if ((id = apic_src_bus_id(apic, pin)) == -1) + goto bad; + + switch (apic_bus_type(id)) { + case ISA: + *flags &= ~IOART_INTALO; /* *flags |= IOART_INTAHI */ + return; + + case EISA: + eirq = apic_src_bus_irq(apic, pin); + if (eirq < 0 || eirq > 15) { + printf("EISA POL: IRQ %d??\n", eirq); + goto bad; + } + /* XXX EISA IRQ's are identical to ISA irq's, regardless of + * whether they are edge or level since they go through the + * level/polarity converter gadget. */ + + if (level == 1) /* XXX Always false */ + pol = 0;/* If level, active low */ + else + pol = 1;/* If edge, high edge */ + + if (pol == 0) + *flags |= IOART_INTALO; + else + *flags &= ~IOART_INTALO; + + return; + + case PCI: + *flags |= IOART_INTALO; + return; + + case -1: + default: + goto bad; + } + +bad: + panic("bad APIC IO INT flags\n"); +} + +/* + * set INT mask bit for each bit set in 'mask'. + * clear INT mask bit for all others. + * only consider lower 24 bits in mask. + */ +#define IO_MASK (imen & 0x00ffffff) +#define IO_FIELD 0x00ffffff + +void +write_io_apic_mask24(int apic, u_int32_t mask) +{ + int x, y; + u_char select; /* the select register is 8 bits */ + u_int32_t low_reg; /* the window register is 32 bits */ + u_int32_t diffs; + + mask &= IO_FIELD; /* safety valve, only use 24 bits */ + if (mask == IO_MASK) /* check for same value as current */ + return; + + diffs = mask ^ IO_MASK; /* record differences */ + + for (x = 0, y = REDIRCNT_IOAPIC(apic); x < y; ++x) { + if (!(diffs & (1 << x))) + continue; /* no change, skip */ + + select = IOAPIC_REDTBL + (x * 2); /* calculate addr */ + low_reg = io_apic_read(apic, select); /* read contents */ + + if (mask & (1 << x)) + low_reg |= IOART_INTMASK; /* set mask */ + else + low_reg &= ~IOART_INTMASK; /* clear mask */ + + io_apic_write(apic, select, low_reg); /* new value */ + } +} + + +#if defined(READY) +/* + * set INT mask bit for each bit set in 'mask'. + * ignore INT mask bit for all others. + * only consider lower 24 bits in mask. + */ +void +set_io_apic_mask24(apic, u_int32_t bits) +{ + int x, y; + u_char select; /* the select register is 8 bits */ + u_int32_t low_reg; /* the window register is 32 bits */ + u_int32_t diffs; + + bits &= IO_FIELD; /* safety valve, only use 24 bits */ + diffs = bits & ~IO_MASK;/* clear AND needing 'set'ing */ + if (!diffs) + return; + +#error imen/IOApicMask NOT merged in setIOApicMask24() + + for (x = 0, y = REDIRCNT_IOAPIC(0); x < y; ++x) { + if (!(diffs & (1 << x))) + continue; /* no change, skip */ + + select = IOAPIC_REDTBL + (x * 2); /* calculate addr */ + low_reg = io_apic_read(apic, select); /* read contents */ + + lowReg |= IOART_INTMASK; /* set mask */ + + io_apic_write(apic, select, low_reg); /* new value */ + } +} +#endif /* READY */ + + +#if defined(READY) +/* + * clear INT mask bit for each bit set in 'mask'. + * ignore INT mask bit for all others. + * only consider lower 24 bits in mask. + */ +void +clr_io_apic_mask24(int apic, u_int32_t bits) +{ + int x, y; + u_char select; /* the select register is 8 bits */ + u_int32_t low_reg; /* the window register is 32 bits */ + u_int32_t diffs; + + bits &= IO_FIELD; /* safety valve, only use 24 bits */ + diffs = bits & IO_MASK; /* set AND needing 'clr'ing */ + if (!diffs) + return; + +#error imen/IOApicMask NOT merged in clrIOApicMask24() + + for (x = 0, y = REDIRCNT_IOAPIC(apic); x < y; ++x) { + if (!(diffs & (1 << x))) + continue; /* no change, skip */ + + select = IOAPIC_REDTBL + (x * 2); /* calculate addr */ + low_reg = io_apic_read(apic, select); /* read contents */ + + low_reg &= ~IOART_INTMASK; /* clear mask */ + + io_apic_write(apic, select, low_reg); /* new value */ + } +} +#endif /* READY */ + +#undef IO_FIELD +#undef IO_MASK + +#endif /* APIC_IO */ + + +/* + * timer code, in development... + * suggested by rgrimes@gndrsh.aac.dev.com + */ + +/** FIXME: temp hack till we can determin bus clock */ +#ifndef BUS_CLOCK +#define BUS_CLOCK 66000000 +#define bus_clock() 66000000 +#endif + +#if defined(READY) + +int acquire_apic_timer __P((void)); +int release_apic_timer __P((void)); + +/* + * acquire the APIC timer for exclusive use + */ +int +acquire_apic_timer(void) +{ +#if 1 + return 0; +#else + /** FIXME: make this really do something */ + panic("APIC timer in use when attempting to aquire\n"); +#endif +} + + +/* + * return the APIC timer + */ +int +release_apic_timer(void) +{ +#if 1 + return 0; +#else + /** FIXME: make this really do something */ + panic("APIC timer was already released\n"); +#endif +} +#endif /* READY */ + + +/* + * load a 'downcount time' in uSeconds + */ +void +set_apic_timer(int value) +{ + u_long lvtt; + long ticks_per_microsec; + + /* calculate divisor and count from value: + * + * timeBase == CPU bus clock divisor == [1,2,4,8,16,32,64,128] value == + * time in uS */ + apic_base[APIC_TDCR] = APIC_TDCR_1; + ticks_per_microsec = bus_clock() / 1000000; + + /* configure timer as one-shot */ + lvtt = apic_base[APIC_LVTT]; + lvtt &= ~(APIC_LVTT_VECTOR | APIC_LVTT_DS | APIC_LVTT_M | APIC_LVTT_TM); + lvtt |= APIC_LVTT_M; /* no INT, one-shot */ + apic_base[APIC_LVTT] = lvtt; + + /* */ + apic_base[APIC_TICR] = value * ticks_per_microsec; +} + + +/* + * read remaining time in timer + */ +int +read_apic_timer(void) +{ +#if 0 + /** FIXME: we need to return the actual remaining time, + * for now we just return the remaining count. + */ +#else + return apic_base[APIC_TCCR]; +#endif +} + + +/* + * spin-style delay, set delay time in uS, spin till it drains + */ +void +u_sleep(int count) +{ + set_apic_timer(count); + while (read_apic_timer()) + /* spin */ ; +} + + +#if defined(TEST_IPI) && defined(IPI_INTS) + +#define TEST_IRQBA (ICU_OFFSET+24) /* BSP & AP */ +#define TEST_IRQB (ICU_OFFSET+25) /* BSP */ +#define TEST_IRQA (ICU_OFFSET+26) /* AP */ + +/** + * a series of tests for the apic IPI routines + * I call them from db (you only get one shot @ the AP, no way to EOI it!) + */ + +#include <sys/param.h> +#include <i386/isa/isa_device.h> +#include <i386/isa/icu.h> + +void apt1(void); +void apt2(void); +void apt3(void); +void apt4(void); +void apt5(void); +void apt6(void); +void apeoi(void); + +void +apt1(void) +{ + selected_procs_ipi(0x0001, TEST_IRQB); /* INT BSP */ +} +void +apt2(void) +{ + selected_procs_ipi(0x0002, TEST_IRQA); /* INT AP */ +} +void +apt3(void) +{ + selected_procs_ipi(0x0003, TEST_IRQBA); /* INT BSP/AP */ +} +void +apt4(void) +{ + all_procs_ipi(TEST_IRQBA); /* INT BSP/AP */ +} +void +apt5(void) +{ + all_but_self_ipi(TEST_IRQA); /* INT AP */ +} +void +apt6(void) +{ + self_ipi(TEST_IRQB); /* INT BSP */ +} +void +apeoi(void) +{ + apic_base[APIC_EOI] = 0; +} +#endif /* TEST_IPI && IPI_INTS */ diff --git a/sys/i386/i386/mpboot.s b/sys/i386/i386/mpboot.s new file mode 100644 index 000000000000..e43b25043b70 --- /dev/null +++ b/sys/i386/i386/mpboot.s @@ -0,0 +1,317 @@ +/* + * Copyright (c) 1995, Jack F. Vogel + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by Jack F. Vogel + * 4. The name of the developer may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * mpboot.s: FreeBSD machine support for the Intel MP Spec + * multiprocessor systems. + * + * $Id: mpboot.s,v 1.18 1997/04/25 03:11:27 fsmp Exp $ + */ + + +#include <machine/asmacros.h> /* miscellaneous asm macros */ +#include <machine/apic.h> +#include <machine/specialreg.h> + +#include "assym.s" + +/* + * this code MUST be enabled here and in mp_machdep.c + * it follows the very early stages of AP boot by placing values in CMOS ram. + * it NORMALLY will never be needed and thus the primitive method for enabling. + * +#define CHECK_POINTS + */ + +#if defined(CHECK_POINTS) + +#define CMOS_REG (0x70) +#define CMOS_DATA (0x71) + +#define CHECKPOINT(A,D) \ + movb $(A),%al ; \ + outb %al,$CMOS_REG ; \ + movb $(D),%al ; \ + outb %al,$CMOS_DATA + +#else + +#define CHECKPOINT(A,D) + +#endif /* CHECK_POINTS */ + + +/* + * the APs enter here from their trampoline code (bootMP, below) + */ + .align 4 + +NON_GPROF_ENTRY(MPentry) + CHECKPOINT(0x36, 3) + movl $mp_stk-KERNBASE,%esp /* mp boot stack end loc. */ + /* Now enable paging mode */ + movl _IdlePTD-KERNBASE, %eax + movl %eax,%cr3 + movl %cr0,%eax + orl $CR0_PE|CR0_PG,%eax /* enable paging */ + movl $0x80000011,%eax + movl %eax,%cr0 /* let the games begin! */ + movl $mp_stk,%esp /* mp boot stack end loc. */ + + pushl $mp_begin /* jump to high mem */ + ret + + /* + * Wait for the booting CPU to signal startup + */ +mp_begin: /* now running relocated at KERNBASE */ + CHECKPOINT(0x37, 4) + call _init_secondary /* load i386 tables */ + CHECKPOINT(0x38, 5) + + /* disable the APIC, just to be SURE */ + movl _apic_base, %esi + movl APIC_SVR(%esi), %eax /* get spurious vector reg. */ + andl $~APIC_SVR_SWEN, %eax /* clear software enable bit */ + movl %eax, APIC_SVR(%esi) + + /* signal our startup to the BSP */ + movl APIC_VER(%esi), %eax /* our version reg contents */ + movl %eax, _cpu_apic_versions /* into [ 0 ] */ + incl _mp_ncpus /* signal BSP */ + + /* One at a time, we are running on the shared mp_stk */ + /* This is the Intel reccomended semaphore method */ +#define BL_SET 0xff +#define BL_CLR 0x00 + movb $BL_SET, %al +1: + xchgb %al, bootlock /* xchg is implicitly locked */ + cmpb $BL_SET, %al /* was is set? */ + jz 1b /* yes, keep trying... */ + CHECKPOINT(0x39, 6) + + /* Now, let's do some REAL WORK :-) */ + call _secondary_main +/* NOT REACHED */ +2: hlt + jmp 2b + +/* + * Let a CPU past the semaphore so it can use mp_stk + */ +ENTRY(boot_unlock) + movb $BL_CLR, %al + xchgb %al, bootlock /* xchg is implicitly locked */ + ret + +/* + * This is the embedded trampoline or bootstrap that is + * copied into 'real-mode' low memory, it is where the + * secondary processor "wakes up". When it is executed + * the processor will eventually jump into the routine + * MPentry, which resides in normal kernel text above + * 1Meg. -jackv + */ + +#define data32 .byte 0x66 +#define addr32 .byte 0x67 + + .data + ALIGN_DATA /* just to be sure */ + +BOOTMP1: + +NON_GPROF_ENTRY(bootMP) + cli + CHECKPOINT(0x34, 1) + /* First guarantee a 'clean slate' */ + data32 + xorl %eax, %eax + data32 + movl %eax, %ebx + data32 + movl %eax, %ecx + data32 + movl %eax, %edx + data32 + movl %eax, %esi + data32 + movl %eax, %edi + + /* set up data segments */ + mov %cs, %ax + mov %ax, %ds + mov %ax, %es + mov %ax, %fs + mov %ax, %gs + mov %ax, %ss + mov $(boot_stk-_bootMP), %sp + + /* Now load the global descriptor table */ + addr32 + data32 + /* XXX: sigh: lgdt MP_GDTptr-_bootMP GAS BUG! */ + .byte 0x0f, 0x01, 0x15 /* XXX hand assemble! */ + .long MP_GDTptr-_bootMP /* XXX hand assemble! */ + + /* Enable protected mode */ + data32 + movl %cr0, %eax + data32 + orl $CR0_PE, %eax + data32 + movl %eax, %cr0 + + /* + * make intrasegment jump to flush the processor pipeline and + * reload CS register + */ + data32 + pushl $0x18 + data32 + pushl $(protmode-_bootMP) + data32 + lret + +protmode: + CHECKPOINT(0x35, 2) + + /* + * we are NOW running for the first time with %eip + * having the full physical address, BUT we still + * are using a segment descriptor with the origin + * not matching the booting kernel. + * + * SO NOW... for the BIG Jump into kernel's segment + * and physical text above 1 Meg. + */ + mov $0x10, %ebx + movw %bx, %ds + movw %bx, %es + movw %bx, %fs + movw %bx, %gs + movw %bx, %ss + + .globl _bigJump +_bigJump: + /* this will be modified by mpInstallTramp() */ + ljmp $0x08, $0 /* far jmp to MPentry() */ + +dead: hlt /* We should never get here */ + jmp dead + +/* + * MP boot strap Global Descriptor Table + */ + .align 4 + .globl _MP_GDT + .globl _bootCodeSeg + .globl _bootDataSeg +_MP_GDT: + +nulldesc: /* offset = 0x0 */ + + .word 0x0 + .word 0x0 + .byte 0x0 + .byte 0x0 + .byte 0x0 + .byte 0x0 + +kernelcode: /* offset = 0x08 */ + + .word 0xffff /* segment limit 0..15 */ + .word 0x0000 /* segment base 0..15 */ + .byte 0x0 /* segment base 16..23; set for 0K */ + .byte 0x9f /* flags; Type */ + .byte 0xcf /* flags; Limit */ + .byte 0x0 /* segment base 24..32 */ + +kerneldata: /* offset = 0x10 */ + + .word 0xffff /* segment limit 0..15 */ + .word 0x0000 /* segment base 0..15 */ + .byte 0x0 /* segment base 16..23; set for 0k */ + .byte 0x93 /* flags; Type */ + .byte 0xcf /* flags; Limit */ + .byte 0x0 /* segment base 24..32 */ + +bootcode: /* offset = 0x18 */ + + .word 0xffff /* segment limit 0..15 */ +_bootCodeSeg: /* this will be modified by mpInstallTramp() */ + .word 0x0000 /* segment base 0..15 */ + .byte 0x00 /* segment base 16...23; set for 0x000xx000 */ + .byte 0x9e /* flags; Type */ + .byte 0xcf /* flags; Limit */ + .byte 0x0 /*segment base 24..32 */ + +bootdata: /* offset = 0x20 */ + + .word 0xffff +_bootDataSeg: /* this will be modified by mpInstallTramp() */ + .word 0x0000 /* segment base 0..15 */ + .byte 0x00 /* segment base 16...23; set for 0x000xx000 */ + .byte 0x92 + .byte 0xcf + .byte 0x0 + +/* + * GDT pointer for the lgdt call + */ + .globl _mp_gdtbase + +MP_GDTptr: +_mp_gdtlimit: + .word 0x0028 +_mp_gdtbase: /* this will be modified by mpInstallTramp() */ + .long 0 + + .space 0x100 /* space for boot_stk - 1st temporary stack */ +boot_stk: + +BOOTMP2: + .globl _bootMP_size +_bootMP_size: + .long BOOTMP2 - BOOTMP1 + + /* + * Temporary stack used while booting AP's + * It is protected by: + * 1: only one cpu is started at a time and it ends up waiting + * for smp_active before continuing. + * 2: Once smp_active != 0; further access is limited by _bootlock. + */ + .globl mp_stk + .space 0x2000 /* space for mp_stk - 2nd temporary stack */ +mp_stk: + + .globl bootlock +bootlock: .byte BL_SET diff --git a/sys/i386/i386/mplock.s b/sys/i386/i386/mplock.s new file mode 100644 index 000000000000..3f413aaa63fe --- /dev/null +++ b/sys/i386/i386/mplock.s @@ -0,0 +1,207 @@ +/* + * ---------------------------------------------------------------------------- + * "THE BEER-WARE LICENSE" (Revision 42): + * <phk@FreeBSD.org> wrote this file. As long as you retain this notice you + * can do whatever you want with this stuff. If we meet some day, and you think + * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp + * ---------------------------------------------------------------------------- + * + * $Id: mplock.s,v 1.16 1997/04/26 06:41:33 peter Exp $ + * + * Functions for locking between CPUs in a SMP system. + * + * This is an "exclusive counting semaphore". This means that it can be + * free (0xffffffff) or be owned by a CPU (0xXXYYYYYY where XX is CPU-id + * and YYYYYY is the count). + * + * Contrary to most implementations around, this one is entirely atomic: + * The attempt to seize/release the semaphore and the increment/decrement + * is done in one atomic operation. This way we are safe from all kinds + * of weird reentrancy situations. + * + */ + +#include "opt_ddb.h" +#include "opt_smp.h" +#include "assym.s" /* system definitions */ +#include <machine/specialreg.h> /* x86 special registers */ +#include <machine/asmacros.h> /* miscellaneous asm macros */ +#include <machine/smpasm.h> +#include <machine/smptests.h> /** TEST_LOPRIO */ + + + .text +/*********************************************************************** + * void MPgetlock(unsigned int *lock) + * ---------------------------------- + * Destroys %eax, %ecx and %edx. + */ + +NON_GPROF_ENTRY(MPgetlock) + movl _smp_active, %eax /* !MP ? -- skip it */ + cmpl $0, %eax + jne 1f + ret +1: movl 4(%esp), %edx /* Get the address of the lock */ + movl (%edx), %eax /* Try to see if we have it already */ + andl $0x00ffffff, %eax /* - get count */ + movl _apic_base, %ecx /* - get cpu# */ + movl APIC_ID(%ecx), %ecx + andl $APIC_ID_MASK, %ecx + orl %ecx, %eax /* - combine them */ + movl %eax, %ecx + incl %ecx /* - new count is one more */ + lock + cmpxchg %ecx, (%edx) /* - try it atomically */ + jne 2f /* - miss */ + ret +2: movl $0xffffffff, %eax /* Assume it's free */ + movl _apic_base, %ecx /* - get cpu# */ + movl APIC_ID(%ecx), %ecx + andl $APIC_ID_MASK, %ecx + incl %ecx /* - new count is one */ + lock + cmpxchg %ecx, (%edx) /* - try it atomically */ + jne 3f /* ...do not collect $200 */ +#if defined(TEST_LOPRIO) + /* 1st acquire, claim LOW PRIO (ie, ALL INTerrupts) */ + movl _apic_base, %ecx /* base addr of LOCAL APIC */ + movl APIC_TPR(%ecx), %eax /* Task Priority Register */ + andl $0xffffff00, %eax /* clear task priority field */ + movl %eax, APIC_TPR(%ecx) /* set it */ +#endif /** TEST_LOPRIO */ + ret +3: cmpl $0xffffffff, (%edx) /* Wait for it to become free */ + jne 3b + jmp 2b /* XXX 1b ? */ + +/*********************************************************************** + * int MPtrylock(unsigned int *lock) + * --------------------------------- + * Destroys %eax, %ecx and %edx. + * Returns 1 if lock was successfull + */ + +NON_GPROF_ENTRY(MPtrylock) + movl _smp_active, %eax /* !MP ? -- skip it */ + cmpl $0, %eax + jne 1f + movl $1, %eax + ret +1: movl 4(%esp), %edx /* Get the address of the lock */ + movl (%edx), %eax /* Try to see if we have it already */ + andl $0x00ffffff, %eax /* - get count */ + movl _apic_base, %ecx /* - get cpu# */ + movl APIC_ID(%ecx), %ecx + andl $APIC_ID_MASK, %ecx + orl %ecx, %eax /* - combine them */ + movl %eax, %ecx + incl %ecx /* - new count is one more */ + lock + cmpxchg %ecx, (%edx) /* - try it atomically */ + jne 2f /* - miss */ + movl $1, %eax + ret +2: movl $0xffffffff, %eax /* Assume it's free */ + movl _apic_base, %ecx /* - get cpu# */ + movl APIC_ID(%ecx), %ecx + andl $APIC_ID_MASK, %ecx + incl %ecx /* - new count is one */ + lock + cmpxchg %ecx, (%edx) /* - try it atomically */ + jne 3f /* ...do not collect $200 */ + movl $1, %eax + ret +3: movl $0, %eax + ret + +/*********************************************************************** + * void MPrellock(unsigned int *lock) + * ---------------------------------- + * Destroys %eax, %ecx and %edx. + */ + +NON_GPROF_ENTRY(MPrellock) + movl _smp_active, %eax /* !MP ? -- skip it */ + cmpl $0, %eax + jne 1f + ret +1: movl 4(%esp), %edx /* Get the address of the lock */ + movl (%edx), %eax /* - get the value */ + movl %eax,%ecx + decl %ecx /* - new count is one less */ + testl $0x00ffffff, %ecx /* - Unless it's zero... */ + jnz 2f +#if defined(TEST_LOPRIO) + /* last release, give up LOW PRIO (ie, arbitrate INTerrupts) */ + movl _apic_base, %ecx /* base addr of LOCAL APIC */ + movl APIC_TPR(%ecx), %eax /* Task Priority Register */ + andl $0xffffff00, %eax /* clear task priority field */ + orl $0x00000010, %eax /* set task priority to 'arbitrate' */ + movl %eax, APIC_TPR(%ecx) /* set it */ + movl (%edx), %eax /* - get the value AGAIN */ +#endif /** TEST_LOPRIO */ + movl $0xffffffff, %ecx /* - In which case we release it */ +2: lock + cmpxchg %ecx, (%edx) /* - try it atomically */ + jne 1b /* ...do not collect $200 */ + ret + +/*********************************************************************** + * void get_mplock() + * ----------------- + * All registers preserved + */ + +NON_GPROF_ENTRY(get_mplock) + pushl %eax + pushl %ecx + pushl %edx + pushl $_mp_lock + call _MPgetlock + add $4, %esp + popl %edx + popl %ecx + popl %eax + ret + +/*********************************************************************** + * void try_mplock() + * ----------------- + * reg %eax == 1 if success + */ + +NON_GPROF_ENTRY(try_mplock) + pushl %ecx + pushl %edx + pushl $_mp_lock + call _MPtrylock + add $4, %esp + popl %edx + popl %ecx + ret + +/*********************************************************************** + * void rel_mplock() + * ----------------- + * All registers preserved + */ + +NON_GPROF_ENTRY(rel_mplock) + pushl %eax + pushl %ecx + pushl %edx + pushl $_mp_lock + call _MPrellock + add $4, %esp + popl %edx + popl %ecx + popl %eax + ret + + + .data + .globl _mp_lock + .align 2 /* mp_lock SHALL be aligned on i386 */ +_mp_lock: .long 0 + diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c new file mode 100644 index 000000000000..07fca34efe56 --- /dev/null +++ b/sys/i386/i386/mptable.c @@ -0,0 +1,1646 @@ +/* + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: mp_machdep.c,v 1.47 1997/04/26 08:11:49 peter Exp $ + */ + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" +#if defined(APIC_IO) +#if !defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#else /* APIC_IO */ +#if defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#endif /* APIC_IO */ +#define FIX_MP_TABLE_WORKS_NOT + +#include "opt_serial.h" + +#include <sys/param.h> /* for KERNBASE */ +#include <sys/types.h> +#include <sys/sysproto.h> +#include <sys/time.h> +#include <sys/systm.h> + +#include <vm/vm.h> /* for KERNBASE */ +#include <vm/vm_param.h> /* for KERNBASE */ +#include <vm/pmap.h> /* for KERNBASE */ +#include <machine/pmap.h> /* for KERNBASE */ + +#include <machine/smp.h> +#include <machine/apic.h> +#include <machine/mpapic.h> +#include <machine/cpufunc.h> +#include <machine/segments.h> +#include <machine/smptests.h> /** TEST_UPPERPRIO, TEST_DEFAULT_CONFIG */ + +#include <i386/i386/cons.h> /* cngetc() */ + +#if defined(IPI_INTS) +#include <i386/isa/isa_device.h> +#include "vector.h" +#endif /* IPI_INTS */ + +#if defined(SMP_INVLTLB) +#include <i386/isa/icu.h> +#endif /* SMP_INVLTLB */ + +#define WARMBOOT_TARGET 0 +#define WARMBOOT_OFF (KERNBASE + 0x0467) +#define WARMBOOT_SEG (KERNBASE + 0x0469) + +#define BIOS_BASE (0xf0000) +#define BIOS_SIZE (0x10000) +#define BIOS_COUNT (BIOS_SIZE/4) + +#define CMOS_REG (0x70) +#define CMOS_DATA (0x71) +#define BIOS_RESET (0x0f) +#define BIOS_WARM (0x0a) + +/* + * this code MUST be enabled here and in mpboot.s. + * it follows the very early stages of AP boot by placing values in CMOS ram. + * it NORMALLY will never be needed and thus the primitive method for enabling. + * +#define CHECK_POINTS + */ + +#if defined(CHECK_POINTS) +#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) +#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) + +#define CHECK_INIT(D); \ + CHECK_WRITE(0x34, (D)); \ + CHECK_WRITE(0x35, (D)); \ + CHECK_WRITE(0x36, (D)); \ + CHECK_WRITE(0x37, (D)); \ + CHECK_WRITE(0x38, (D)); \ + CHECK_WRITE(0x39, (D)); + +#define CHECK_PRINT(S); \ + printf("%s: %d, %d, %d, %d, %d, %d\n", \ + (S), \ + CHECK_READ(0x34), \ + CHECK_READ(0x35), \ + CHECK_READ(0x36), \ + CHECK_READ(0x37), \ + CHECK_READ(0x38), \ + CHECK_READ(0x39)); + +#else /* CHECK_POINTS */ + +#define CHECK_INIT(D) +#define CHECK_PRINT(S) + +#endif /* CHECK_POINTS */ + + +/** FIXME: what system files declare these??? */ +extern struct region_descriptor r_gdt, r_idt; + +/* global data */ +struct proc *SMPcurproc[NCPU]; +struct pcb *SMPcurpcb[NCPU]; +struct timeval SMPruntime[NCPU]; + +int mp_ncpus; /* # of CPUs, including BSP */ +int mp_naps; /* # of Applications processors */ +int mp_nbusses; /* # of busses */ +int mp_napics; /* # of IO APICs */ +int mpenabled; +int boot_cpu_id; /* designated BSP */ +vm_offset_t cpu_apic_address; +vm_offset_t io_apic_address[NAPIC]; + +u_int32_t cpu_apic_versions[NCPU]; +u_int32_t io_apic_versions[NAPIC]; + +/* + * APIC ID logical/physical mapping structures + */ +int cpu_num_to_apic_id[NCPU]; +int io_num_to_apic_id[NAPIC]; +int apic_id_to_logical[NAPICID]; + +/* + * look for MP compliant motherboard. + */ + +static u_int boot_address; +static u_int base_memory; + +static int picmode; /* 0: virtual wire mode, 1: PIC mode */ +static u_int mpfps; +static int search_for_sig(u_int32_t target, int count); +static int mp_probe(u_int base_top); +static void mp_enable(u_int boot_addr); + +#if defined(IPI_INTS) +static void ipi_intr0(void); +static void ipi_intr1(void); +static void ipi_intr2(void); +static void ipi_intr3(void); +static int +ipi_ihandler_attach(int irq, inthand2_t * func, + unsigned *maskptr, int unit); +#endif /* IPI_INTS */ + + +/* + * calculate usable address in base memory for AP trampoline code + */ +u_int +mp_bootaddress(u_int basemem) +{ + base_memory = basemem * 1024; /* convert to bytes */ + + boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ + if ((base_memory - boot_address) < bootMP_size) + boot_address -= 4096; /* not enough, lower by 4k */ + + return boot_address; +} + + +/* + * startup the SMP processors + */ +void +mp_start(void) +{ + /* look for MP capable motherboard */ + if (mp_probe(base_memory)) + mp_enable(boot_address); + else { + printf("MP FPS NOT FOUND, suggest use of 'mptable' program\n"); + panic("can't continue!\n"); + } + + /* finish pmap initialization - turn off V==P mapping at zero */ + pmap_bootstrap2(); +} + + +/* + * print various information about the SMP system hardware and setup + */ +void +mp_announce(void) +{ + int x; + + printf("FreeBSD/SMP: Multiprocessor motherboard\n"); + printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); + printf(", version: 0x%08x\n", cpu_apic_versions[0]); + for (x = 1; x <= mp_naps; ++x) { + printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); + printf(", version: 0x%08x\n", cpu_apic_versions[x]); + } + +#if defined(APIC_IO) + for (x = 0; x < mp_napics; ++x) { + printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); + printf(", version: 0x%08x\n", io_apic_versions[x]); + } +#else + printf(" Warning: APIC I/O disabled\n"); +#endif /* APIC_IO */ +} + + +/* + * AP cpu's call this to sync up protected mode. + */ +void +init_secondary(void) +{ + int gsel_tss, slot; + + r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; + r_gdt.rd_base = (int) gdt; + lgdt(&r_gdt); /* does magic intra-segment return */ + lidt(&r_idt); + lldt(_default_ldt); + + slot = NGDT + cpunumber(); + gsel_tss = GSEL(slot, SEL_KPL); + gdt[slot].sd.sd_type = SDT_SYS386TSS; + ltr(gsel_tss); + + load_cr0(0x8005003b); /* XXX! */ +} + + +#if defined(APIC_IO) +void +configure_local_apic(void) +{ + u_char byte; + u_int32_t temp; + + if (picmode) { + outb(0x22, 0x70); /* select IMCR */ + byte = inb(0x23); /* current contents */ + byte |= 0x01; /* mask external INTR */ + outb(0x23, byte); /* disconnect 8259s/NMI */ + } + /* mask the LVT1 */ + temp = apic_base[APIC_LVT1]; + temp |= APIC_LVT_M; + apic_base[APIC_LVT1] = temp; +} +#endif /* APIC_IO */ + + +/******************************************************************* + * local functions and data + */ + +static int +mp_probe(u_int base_top) +{ + int x; + u_long segment; + u_int32_t target; + + /* see if EBDA exists */ + if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { + /* search first 1K of EBDA */ + target = (u_int32_t) (segment << 4); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } else { + /*last 1K of base memory, effective 'top of base' is passed in*/ + target = (u_int32_t) (base_top - 0x400); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } + + /* search the BIOS */ + target = (u_int32_t) BIOS_BASE; + if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) + goto found; + + /* nothing found */ + mpfps = mpenabled = 0; + return 0; + +found: /* please forgive the 'goto'! */ + /* flag fact that we are running multiple processors */ + mpfps = x; + mpenabled = 1; + return 1; +} + + +/* + * start the SMP system + */ +static int parse_mp_table(void); +static void default_mp_table(int type); +static int start_all_aps(u_int boot_addr); + +#if defined(XFAST_IPI32) +#include <machine/md_var.h> +#include <i386/isa/isa_device.h> +extern void Xfastipi32(u_int, u_int, u_int, u_int); +#endif /* XFAST_IPI32 */ + +static void +mp_enable(u_int boot_addr) +{ + int x; +#if defined(APIC_IO) + int apic; + u_int ux; +#if defined(TEST_UPPERPRIO) + u_char select; /* the select register is 8 bits */ + u_int32_t flags; /* the window register is 32 bits */ +#endif /* TEST_UPPERPRIO */ +#endif /* APIC_IO */ + + /* examine the MP table for needed info */ + x = parse_mp_table(); + + /* create pages for (address common) cpu APIC and each IO APIC */ + pmap_bootstrap_apics(); + + /* can't process default configs till the CPU APIC is pmapped */ + if (x) + default_mp_table(x); + +#if defined(APIC_IO) + /* fill the LOGICAL io_apic_versions table */ + for (apic = 0; apic < mp_napics; ++apic) { + ux = io_apic_read(apic, IOAPIC_VER); + io_apic_versions[apic] = ux; + } + + /* + */ + for (apic = 0; apic < mp_napics; ++apic) + if (io_apic_setup(apic) < 0) + panic("IO APIC setup failure\n"); + +#if defined(IPI_INTS) + /* setup IPI INTerrupt mechanism */ + ipi_ihandler_attach( /* irq */ 24, + /* XXX */ (inthand2_t *) ipi_intr0, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 25, + /* XXX */ (inthand2_t *) ipi_intr1, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 26, + /* XXX */ (inthand2_t *) ipi_intr2, NULL, /* unit */ 0); +#if !defined(SMP_INVLTLB) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); +#else +#if defined(XFAST_IPI32) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); + setidt(ICU_OFFSET + 32, + Xfastipi32, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#else + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_invltlb, NULL, /* unit */ 0); +#endif /* XFAST_IPI32 */ +#endif + +#endif /* IPI_INTS */ + +#if defined(TEST_UPPERPRIO) + +#if 1 + printf("special IRQ10\n"); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~0xff; /** clear vector */ + flags |= 64; + io_apic_write(0, select, flags); +#else + printf("special IRQ10\n"); + cngetc(); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~IOART_DELMOD; /* FIXED mode */ + io_apic_write(0, select, flags); + io_apic_write(0, select + 1, boot_cpu_id << 24); +#endif /** 0/1 */ + +#endif /* TEST_UPPERPRIO */ + +#endif /* APIC_IO */ + + /* start each Application Processor */ + start_all_aps(boot_addr); +} + + +/* + * look for the MP spec signature + */ + +/* string defined by the Intel MP Spec as identifying the MP table */ +#define MP_SIG 0x5f504d5f /* _MP_ */ +#define NEXT(X) ((X) += 4) +static int +search_for_sig(u_int32_t target, int count) +{ + int x; + u_int32_t *addr = (u_int32_t *) (KERNBASE + target); + + for (x = 0; x < count; NEXT(x)) + if (addr[x] == MP_SIG) + /* make array index a byte index */ + return (target + (x * sizeof(u_int32_t))); + + return -1; +} + + +#define PROCENTRY_FLAG_EN 0x01 +#define PROCENTRY_FLAG_BP 0x02 +#define IOAPICENTRY_FLAG_EN 0x01 + +/* MP Floating Pointer Structure */ +typedef struct MPFPS { + char signature[4]; + void *pap; + u_char length; + u_char spec_rev; + u_char checksum; + u_char mpfb1; + u_char mpfb2; + u_char mpfb3; + u_char mpfb4; + u_char mpfb5; +} *mpfps_t; +/* MP Configuration Table Header */ +typedef struct MPCTH { + char signature[4]; + u_short base_table_length; + u_char spec_rev; + u_char checksum; + u_char oem_id[8]; + u_char product_id[12]; + void *oem_table_pointer; + u_short oem_table_size; + u_short entry_count; + void *apic_address; + u_short extended_table_length; + u_char extended_table_checksum; + u_char reserved; +} *mpcth_t; + + +typedef struct PROCENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char cpu_flags; + u_long cpu_signature; + u_long feature_flags; + u_long reserved1; + u_long reserved2; +} *proc_entry_ptr; + +typedef struct BUSENTRY { + u_char type; + u_char bus_id; + char bus_type[6]; +} *bus_entry_ptr; + +typedef struct IOAPICENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char apic_flags; + void *apic_address; +} *io_apic_entry_ptr; + +typedef struct INTENTRY { + u_char type; + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} *int_entry_ptr; +/* descriptions of MP basetable entries */ +typedef struct BASETABLE_ENTRY { + u_char type; + u_char length; + char name[16]; +} basetable_entry; + +static basetable_entry basetable_entry_types[] = +{ + {0, 20, "Processor"}, + {1, 8, "Bus"}, + {2, 8, "I/O APIC"}, + {3, 8, "I/O INT"}, + {4, 8, "Local INT"} +}; + +typedef struct BUSDATA { + u_char bus_id; + enum busTypes bus_type; +} bus_datum; + +typedef struct INTDATA { + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} io_int, local_int; + +typedef struct BUSTYPENAME { + u_char type; + char name[7]; +} bus_type_name; + +static bus_type_name bus_type_table[] = +{ + {CBUS, "CBUS"}, + {CBUSII, "CBUSII"}, + {EISA, "EISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {ISA, "ISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {PCI, "PCI"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {XPRESS, "XPRESS"}, + {UNKNOWN_BUSTYPE, "---"} +}; +/* from MP spec v1.4, table 5-1 */ +static int default_data[7][5] = +{ +/* nbus, id0, type0, id1, type1 */ + {1, 0, ISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {0, 255, 255, 255, 255},/* MCA not supported */ + {2, 0, ISA, 1, PCI}, + {2, 0, EISA, 1, PCI}, + {0, 255, 255, 255, 255} /* MCA not supported */ +}; + + +/* the bus data */ +bus_datum bus_data[NBUS]; + +/* the IO INT data, one entry per possible APIC INTerrupt */ +io_int io_apic_ints[NINTR]; + +static int nintrs; + +#if defined(FIX_MP_TABLE_WORKS) +static void fix_mp_table __P((void)); +#endif /* FIX_MP_TABLE_WORKS */ + +static void processor_entry __P((proc_entry_ptr entry, int *cpu)); +static void io_apic_entry __P((io_apic_entry_ptr entry, int *apic)); +static void bus_entry __P((bus_entry_ptr entry, int *bus)); +static void int_entry __P((int_entry_ptr entry, int *intr)); +static int lookup_bus_type __P((char *name)); + + +/* + * parse an Intel MP specification table + */ +static int +parse_mp_table(void) +{ + int x; + mpfps_t fps; + mpcth_t cth; + int totalSize; + void *position; + int count; + int type; + int apic, bus, cpu, intr; + + /* clear physical APIC ID to logical CPU/IO table */ + for (x = 0; x < NAPICID; ++x) + ID_TO_IO(x) = -1; + + /* clear logical CPU to APIC ID table */ + for (x = 0; x < NCPU; ++x) + CPU_TO_ID(x) = -1; + + /* clear logical IO to APIC ID table */ + for (x = 0; x < NAPIC; ++x) + IO_TO_ID(x) = -1; + + /* clear IO APIC address table */ + for (x = 0; x < NAPIC; ++x) + io_apic_address[x] = ~0; + + /* clear bus data table */ + for (x = 0; x < NBUS; ++x) + bus_data[x].bus_id = 0xff; + + /* clear IO APIC INT table */ + for (x = 0; x < NINTR; ++x) + io_apic_ints[x].int_type = 0xff; + nintrs = 0; + + /* count the BSP */ + mp_ncpus = 1; + + /* setup the cpu/apic mapping arrays */ + boot_cpu_id = -1; + + /* local pointer */ + fps = (mpfps_t) mpfps; + + /* record whether PIC or virtual-wire mode */ + picmode = (fps->mpfb2 & 0x80) ? 1 : 0; + + /* check for use of 'default' configuration */ +#if defined(TEST_DEFAULT_CONFIG) + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return TEST_DEFAULT_CONFIG; +#else + if (fps->mpfb1 != 0) { + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return fps->mpfb1; + } +#endif /* TEST_DEFAULT_CONFIG */ + + if ((cth = fps->pap) == 0) + panic("MP Configuration Table Header MISSING!\n"); + + cpu_apic_address = (vm_offset_t) cth->apic_address; + + totalSize = cth->base_table_length - sizeof(struct MPCTH); + position = (u_char *) cth + sizeof(struct MPCTH); + count = cth->entry_count; + + apic = 0; /* logical apic# start @ 0 */ + bus = 0; /* logical bus# start @ 0 */ + cpu = 1; /* logical cpu# start @ 0, BUT reserve 0 for */ + /* BSP */ + intr = 0; /* unknown */ + + /* walk the table, recording info of interest */ + while (count--) { + switch (type = *(u_char *) position) { + case 0: + processor_entry(position, &cpu); + break; + case 1: + bus_entry(position, &bus); + break; + case 2: + io_apic_entry(position, &apic); + break; + case 3: + int_entry(position, &intr); + break; + case 4: + /* int_entry(position); */ + break; + default: + panic("mpfps Base Table HOSED!\n"); + /* NOTREACHED */ + } + + totalSize -= basetable_entry_types[type].length; + (u_char *) position += basetable_entry_types[type].length; + } + + if (boot_cpu_id == -1) + panic("NO BSP found!\n"); + + /* record # of APs found */ + mp_naps = (cpu - 1); + + /* record # of busses found */ + mp_nbusses = bus; + + /* record # of IO APICs found */ + mp_napics = apic; + + /* record # of IO APICs found */ + nintrs = intr; + +#if defined(FIX_MP_TABLE_WORKS) + /* post scan cleanup */ + fix_mp_table(); +#endif /* FIX_MP_TABLE_WORKS */ + + /* report fact that its NOT a default configuration */ + return 0; +} + + +/* + * parse an Intel MP specification table + */ +#if defined(FIX_MP_TABLE_WORKS) +static void +fix_mp_table(void) +{ + int x; + int y; + int num_pci_bus; + bus_datum bus_record; + + /* + * Fix mis-numbering of the PCI bus and its INT entries if the BIOS + * did it wrong. The MP spec says that when more than 1 PCI bus + * exists the BIOS must begin with bus entries for the PCI bus and use + * actual PCI bus numbering. This implies that when only 1 PCI bus + * exists the BIOS can choose to ignore this ordering, and indeed many + * MP motherboards do ignore it. This causes a problem when the PCI + * sub-system makes requests of the MP sub-system based on PCI bus + * numbers. So here we look for the situation and renumber the + * busses and associated INTs in an effort to "make it right". + */ + + /* count the number of PCI busses */ + for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + ++num_pci_bus; + } + + /* check the 1 PCI bus case for sanity */ + if (num_pci_bus == 1) { + + /* if its in the first slot all is well */ + if (bus_data[0].bus_type == PCI) + return; + + /* mis-numbered, swap with whichever bus uses slot 0 */ + + /* locate the entry holding the PCI bus */ + for (x = 1; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + break; + } + + /* swap the bus entry records */ + bus_record = bus_data[0]; + bus_data[0] = bus_data[x]; + bus_data[x] = bus_record; + + /* swap each relavant INTerrupt entry */ + for (y = 0; y < nintrs; ++y) { + if (io_apic_ints[y].src_bus_id == x) + io_apic_ints[y].src_bus_id = 0; + else + if (io_apic_ints[y].src_bus_id == 0) + io_apic_ints[y].src_bus_id = x; + } + } + /* sanity check if more than 1 PCI bus */ + else + if (num_pci_bus > 1) { + for (x = 0; x < num_pci_bus; ++x) { + if (bus_data[x].bus_type != PCI) { + printf("bad PCI bus numbering\n"); + panic("\n"); + } + } + } +} +#endif /* FIX_MP_TABLE_WORKS */ + + +static void +processor_entry(proc_entry_ptr entry, int *cpu) +{ + int x = *cpu; + + /* check for usability */ + if (!(entry->cpu_flags & PROCENTRY_FLAG_EN)) + return; + + /* check for BSP flag */ + if (entry->cpu_flags & PROCENTRY_FLAG_BP) { + /* always give boot CPU the logical value of 0 */ + x = 0; + boot_cpu_id = entry->apic_id; + } else { + /* add another AP to list, if less than max number of CPUs */ + if (x == NCPU) { + printf("Warning: only using %d of the available CPUs!\n", x); + return; + } + ++(*cpu); + } + + CPU_TO_ID(x) = entry->apic_id; + ID_TO_CPU(entry->apic_id) = x; +} + + +static void +bus_entry(bus_entry_ptr entry, int *bus) +{ + int x, y; + char name[8]; + char c; + + if ((x = (*bus)++) == NBUS) + panic("too many busses, increase 'NBUS'\n"); + + /* encode the name into an index */ + for (y = 0; y < 6; ++y) { + if ((c = entry->bus_type[y]) == ' ') + break; + name[y] = c; + } + name[y] = '\0'; + + if ((y = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) + panic("unknown bus type: '%s'\n", name); + + bus_data[x].bus_id = entry->bus_id; + bus_data[x].bus_type = y; +} + + +static void +io_apic_entry(io_apic_entry_ptr entry, int *apic) +{ + int x; + + if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) + return; + + if ((x = (*apic)++) == NAPIC) + panic("too many APICs, increase 'NAPIC'\n"); + + IO_TO_ID(x) = entry->apic_id; + ID_TO_IO(entry->apic_id) = x; + + io_apic_address[x] = (vm_offset_t) entry->apic_address; +} + + +static int +lookup_bus_type(char *name) +{ + int x; + + for (x = 0; x < MAX_BUSTYPE; ++x) + if (strcmp(bus_type_table[x].name, name) == 0) + return bus_type_table[x].type; + + return UNKNOWN_BUSTYPE; +} + + +static void +int_entry(int_entry_ptr entry, int *intr) +{ + int x; + + if ((x = (*intr)++) == NINTR) + panic("too many INTs, increase 'NINTR'\n"); + + io_apic_ints[x].int_type = entry->int_type; + io_apic_ints[x].int_flags = entry->int_flags; + io_apic_ints[x].src_bus_id = entry->src_bus_id; + io_apic_ints[x].src_bus_irq = entry->src_bus_irq; + io_apic_ints[x].dst_apic_id = entry->dst_apic_id; + io_apic_ints[x].dst_apic_int = entry->dst_apic_int; +} + + +static int +apic_int_is_bus_type(int intr, int bus_type) +{ + int bus; + + for (bus = 0; bus < mp_nbusses; ++bus) + if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) + && ((int) bus_data[bus].bus_type == bus_type)) + return 1; + + return 0; +} + + +/* + * determine which APIC pin an ISA INT is attached to. + */ +#define INTTYPE(I) (io_apic_ints[(I)].int_type) +#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) + +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_isa_apic_irq(int isaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (isaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == isaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, ISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin an EISA INT is attached to. + */ +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_eisa_apic_irq(int eisaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (eisaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == eisaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, EISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin a PCI INT is attached to. + */ +#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) +#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) +#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) +int +get_pci_apic_irq(int pciBus, int pciDevice, int pciInt) +{ + int intr; + + --pciInt; /* zero based */ + + for (intr = 0; intr < nintrs; ++intr) /* search each record */ + if ((INTTYPE(intr) == 0) +#if defined(FIX_MP_TABLE_WORKS) + && (SRCBUSID(intr) == pciBus) +#endif /* FIX_MP_TABLE_WORKS */ + && (SRCBUSDEVICE(intr) == pciDevice) + && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, PCI)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSLINE +#undef SRCBUSDEVICE +#undef SRCBUSID + +#undef INTPIN +#undef INTTYPE + + +int +undirect_pci_irq(int rirq) +{ +#if defined(READY) + printf("Freeing irq %d for ISA cards.\n", rirq); + /** FIXME: tickle the MB redirector chip */ + return ???; +#else + printf("Freeing (NOT implimented) irq %d for ISA cards.\n", rirq); + return 0; +#endif /* READY */ +} + + +/* + * given a bus ID, return: + * the bus type if found + * -1 if NOT found + */ +int +apic_bus_type(int id) +{ + int x; + + for (x = 0; x < mp_nbusses; ++x) + if (bus_data[x].bus_id == id) + return bus_data[x].bus_type; + + return -1; +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus ID if found + * -1 if NOT found + */ +int +apic_src_bus_id(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_id); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus IRQ if found + * -1 if NOT found + */ +int +apic_src_bus_irq(int apic, int pin) +{ + int x; + + for (x = 0; x < nintrs; x++) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_irq); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated INTerrupt type if found + * -1 if NOT found + */ +int +apic_int_type(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_type); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated trigger mode if found + * -1 if NOT found + */ +int +apic_trigger(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return ((io_apic_ints[x].int_flags >> 2) & 0x03); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated 'active' level if found + * -1 if NOT found + */ +int +apic_polarity(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_flags & 0x03); + + return -1; /* NOT found */ +} + + +/* + * set data according to MP defaults + * FIXME: probably not complete yet... + */ +static void +default_mp_table(int type) +{ + int ap_cpu_id; +#if defined(APIC_IO) + u_int32_t ux; + int io_apic_id; + int pin; +#endif /* APIC_IO */ + +#if 0 + printf(" MP default config type: %d\n", type); + switch (type) { + case 1: + printf(" bus: ISA, APIC: 82489DX\n"); + break; + case 2: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 3: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 4: + printf(" bus: MCA, APIC: 82489DX\n"); + break; + case 5: + printf(" bus: ISA+PCI, APIC: Integrated\n"); + break; + case 6: + printf(" bus: EISA+PCI, APIC: Integrated\n"); + break; + case 7: + printf(" bus: MCA+PCI, APIC: Integrated\n"); + break; + default: + printf(" future type\n"); + break; + /* NOTREACHED */ + } +#endif /* 0 */ + + boot_cpu_id = (apic_base[APIC_ID] & APIC_ID_MASK) >> 24; + ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; + + /* BSP */ + CPU_TO_ID(0) = boot_cpu_id; + ID_TO_CPU(boot_cpu_id) = 0; + + /* one and only AP */ + CPU_TO_ID(1) = ap_cpu_id; + ID_TO_CPU(ap_cpu_id) = 1; + mp_naps = 1; + + /* one and only IO APIC */ +#if defined(APIC_IO) + io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; + + /* + * sanity check, refer to MP spec section 3.6.6, last paragraph + * necessary as some hardware isn't properly setting up the IO APIC + */ +#if defined(REALLY_ANAL_IOAPICID_VALUE) + if (io_apic_id != 2) { +#else + if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { +#endif /* REALLY_ANAL_IOAPICID_VALUE */ + ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ + ux &= ~APIC_ID_MASK; /* clear the ID field */ + ux |= 0x02000000; /* set it to '2' */ + io_apic_write(0, IOAPIC_ID, ux); /* write new value */ + ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ + if ((ux & APIC_ID_MASK) != 0x02000000) + panic("Problem: can't control IO APIC ID, reg: 0x%08x\n", ux); + io_apic_id = 2; + } + IO_TO_ID(0) = io_apic_id; + ID_TO_IO(io_apic_id) = 0; + mp_napics = 1; +#else + mp_napics = 0; +#endif /* APIC_IO */ + + /* fill out bus entries */ + switch (type) { + case 1: + case 2: + case 3: + case 5: + case 6: + mp_nbusses = default_data[type - 1][0]; + bus_data[0].bus_id = default_data[type - 1][1]; + bus_data[0].bus_type = default_data[type - 1][2]; + bus_data[1].bus_id = default_data[type - 1][3]; + bus_data[1].bus_type = default_data[type - 1][4]; + break; + + /* case 4: case 7: MCA NOT supported */ + default: /* illegal/reserved */ + panic("BAD default MP config: %d\n", type); + } + +#if defined(APIC_IO) + /* general cases from MP v1.4, table 5-2 */ + for (pin = 0; pin < 16; ++pin) { + io_apic_ints[pin].int_type = 0; + io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ + io_apic_ints[pin].src_bus_id = 0; + io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ + io_apic_ints[pin].dst_apic_id = io_apic_id; + io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ + } + + /* special cases from MP v1.4, table 5-2 */ + if (type == 2) { + io_apic_ints[2].int_type = 0xff; /* N/C */ + io_apic_ints[13].int_type = 0xff; /* N/C */ +#if !defined(APIC_MIXED_MODE) + /** FIXME: ??? */ + panic("sorry, can't support type 2 default yet\n"); +#endif /* APIC_MIXED_MODE */ + } else + io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ + + if (type == 7) + io_apic_ints[0].int_type = 0xff; /* N/C */ + else + io_apic_ints[0].int_type = 3; /* vectored 8259 */ + + nintrs = 16; +#endif /* APIC_IO */ +} + + +static void install_ap_tramp(u_int boot_addr); +static int start_ap(int logicalCpu, u_int boot_addr); + +/* + * start each AP in our list + */ +static int +start_all_aps(u_int boot_addr) +{ + int x; + u_char mpbiosreason; + u_long mpbioswarmvec; + + /** + * NOTE: this needs further thought: + * where does it get released? + * should it be set to empy? + * + * get the initial mp_lock with a count of 1 for the BSP + */ + mp_lock = (apic_base[APIC_ID] & APIC_ID_MASK) + 1; + + /* initialize BSP's local APIC */ + apic_initialize(1); + + /* install the AP 1st level boot code */ + install_ap_tramp(boot_addr); + + /* save the current value of the warm-start vector */ + mpbioswarmvec = *((u_long *) WARMBOOT_OFF); + outb(CMOS_REG, BIOS_RESET); + mpbiosreason = inb(CMOS_DATA); + + /* start each AP */ + for (x = 1; x <= mp_naps; ++x) { + + /* setup a vector to our boot code */ + *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; + *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ + + /* attempt to start the Application Processor */ + CHECK_INIT(99); /* setup checkpoints */ + if (!start_ap(x, boot_addr)) { + printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); + CHECK_PRINT("trace"); /* show checkpoints */ + /* + * better panic as the AP may be running loose + * somewhere + */ + printf("panic y/n? [n] "); + if (cngetc() != 'n') + panic("bye-bye\n"); + } + CHECK_PRINT("trace"); /* show checkpoints */ + + /* record its version info */ + cpu_apic_versions[x] = cpu_apic_versions[0]; + } + + /* fill in our (BSP) APIC version */ + cpu_apic_versions[0] = apic_base[APIC_VER]; + + /* restore the warmstart vector */ + *(u_long *) WARMBOOT_OFF = mpbioswarmvec; + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, mpbiosreason); + + /* number of APs actually started */ + return mp_ncpus - 1; +} + + +/* + * load the 1st level AP boot code into base memory. + */ + +/* targets for relocation */ +extern void bigJump(void); +extern void bootCodeSeg(void); +extern void bootDataSeg(void); +extern void MPentry(void); +extern u_int MP_GDT; +extern u_int mp_gdtbase; + +static void +install_ap_tramp(u_int boot_addr) +{ + int x; + int size = *(int *) ((u_long) & bootMP_size); + u_char *src = (u_char *) ((u_long) bootMP); + u_char *dst = (u_char *) boot_addr + KERNBASE; + u_int boot_base = (u_int) bootMP; + u_int8_t *dst8; + u_int16_t *dst16; + u_int32_t *dst32; + + for (x = 0; x < size; ++x) + *dst++ = *src++; + + /* + * modify addresses in code we just moved to basemem. unfortunately we + * need fairly detailed info about mpboot.s for this to work. changes + * to mpboot.s might require changes here. + */ + + /* boot code is located in KERNEL space */ + dst = (u_char *) boot_addr + KERNBASE; + + /* modify the lgdt arg */ + dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); + *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); + + /* modify the ljmp target for MPentry() */ + dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); + *dst32 = ((u_int) MPentry - KERNBASE); + + /* modify the target for boot code segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; + + /* modify the target for boot data segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; +} + + +/* + * this function starts the AP (application processor) identified + * by the APIC ID 'physicalCpu'. It does quite a "song and dance" + * to accomplish this. This is necessary because of the nuances + * of the different hardware we might encounter. It ain't pretty, + * but it seems to work. + */ +static int +start_ap(int logical_cpu, u_int boot_addr) +{ + int physical_cpu; + int vector; + int cpus; + u_long icr_lo, icr_hi; + + /* get the PHYSICAL APIC ID# */ + physical_cpu = CPU_TO_ID(logical_cpu); + + /* calculate the vector */ + vector = (boot_addr >> 12) & 0xff; + + /* used as a watchpoint to signal AP startup */ + cpus = mp_ncpus; + + /* + * first we do an INIT/RESET IPI this INIT IPI might be run, reseting + * and running the target CPU. OR this INIT IPI might be latched (P5 + * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be + * ignored. + */ + + /* setup the address for the target AP */ + icr_hi = apic_base[APIC_ICR_HI] & ~APIC_ID_MASK; + icr_hi |= (physical_cpu << 24); + apic_base[APIC_ICR_HI] = icr_hi; + + /* do an INIT IPI: assert RESET */ + icr_lo = apic_base[APIC_ICR_LOW] & 0xfff00000; + apic_base[APIC_ICR_LOW] = icr_lo | 0x0000c500; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* do an INIT IPI: deassert RESET */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00008500; + + /* wait for pending status end */ + u_sleep(10000); /* wait ~10mS */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* + * next we do a STARTUP IPI: the previous INIT IPI might still be + * latched, (P5 bug) this 1st STARTUP would then terminate + * immediately, and the previously started INIT IPI would continue. OR + * the previous INIT IPI has already run. and this STARTUP IPI will + * run. OR the previous INIT IPI was ignored. and this STARTUP IPI + * will run. + */ + + /* do a STARTUP IPI */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* + * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF + * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR + * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is + * recognized after hardware RESET or INIT IPI. + */ + + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* wait for it to start */ + set_apic_timer(5000000);/* == 5 seconds */ + while (read_apic_timer()) + if (mp_ncpus > cpus) + return 1; /* return SUCCESS */ + + return 0; /* return FAILURE */ +} + + +#if defined(IPI_INTS) + +static void +ipi_intr0(void) +{ + printf("IPI 0\n"); +} + + +static void +ipi_intr1(void) +{ + printf("IPI 1\n"); +} + + +static void +ipi_intr2(void) +{ + printf("IPI 2\n"); +} + + +static void +ipi_intr3(void) +{ + printf("IPI 3\n"); +} + +/*----------------------------------------------------------------------- +** +** Register an interupt handler for an IPI. +** (Stolen from the PCI<->ISA glue code) +** +**----------------------------------------------------------------------- +*/ + +static int +ipi_ihandler_attach(int irq, inthand2_t * func, unsigned *maskptr, int unit) +{ + char buf[16]; + char *cp; + int free_id, id, result; + + sprintf(buf, "ipi irq%d", irq); + for (cp = intrnames, free_id = 0, id = 0; id < NR_DEVICES; id++) { + if (strcmp(cp, buf) == 0) + break; + if (free_id <= 0 && strcmp(cp, "ipi irqnn") == 0) + free_id = id; + while (*cp++ != '\0'); + } + if (id == NR_DEVICES) { + id = free_id; + if (id == 0) { + /* + * All ipi irq counters are in use, perhaps because + * config is old so there aren't any. Abuse the clk0 + * counter. + */ + printf( + "ipi_ihandler_attach: counting ipi irq%d's as clk0 irqs\n", + irq); + } + } + result = register_intr( + irq, /* isa irq */ + id, /* device id */ + 0, /* flags? */ + func, /* handler */ + maskptr, /* mask pointer */ + unit); /* handler arg */ + + if (result) { + printf("WARNING: ipi_ihandler_attach: result=%d\n", result); + return (result); + }; + + return (0); +} +#endif /* IPI_INTS */ + + +#ifdef SMP_INVLTLB +/* + * Flush the TLB on all other CPU's + * + * XXX: Needs to handshake and wait for completion before proceding. + */ + +void +smp_invltlb() +{ + if (smp_active) { + if (invldebug & 2) +#if defined(XFAST_IPI32) + all_but_self_ipi(ICU_OFFSET + 32); +#else + all_but_self_ipi(ICU_OFFSET + 27); +#endif /* XFAST_IPI32 */ + } +} + +void +invlpg(u_int addr) +{ + __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); + smp_invltlb(); +} + +void +invltlb(void) +{ + u_long temp; + /* + * This should be implemented as load_cr3(rcr3()) when load_cr3() is + * inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); + smp_invltlb(); +} +/* + * Handles recieving an "IRQ 27", the invalidate tlb IPI.. + */ +void +ipi_invltlb(void) +{ + u_long temp; + + if (invldebug & 4) { + /* + * This should be implemented as load_cr3(rcr3()) when + * load_cr3() is inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) + :: "memory"); + + } +} +#endif /* SMP_INVLTLB */ diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index 26089cc4fdbc..3f218646416e 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -39,7 +39,7 @@ * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $Id: pmap.c,v 1.140 1997/04/13 01:48:08 dyson Exp $ + * $Id: pmap.c,v 1.141 1997/04/13 03:35:30 dyson Exp $ */ /* @@ -68,6 +68,7 @@ * and to when physical maps must be made correct. */ +#include "opt_smp.h" #include "opt_cpu.h" #define PMAP_LOCK 1 @@ -96,10 +97,16 @@ #include <sys/user.h> +#include <machine/cpu.h> #include <machine/pcb.h> #include <machine/cputypes.h> #include <machine/md_var.h> #include <machine/specialreg.h> +#if defined(SMP) || defined(APIC_IO) +#include <machine/smp.h> +#include <machine/apic.h> +#include <machine/smptests.h> +#endif /* SMP || APIC_IO */ #define PMAP_KEEP_PDIRS #define PMAP_SHPGPERPROC 200 @@ -159,7 +166,6 @@ static vm_page_t nkpg; vm_offset_t kernel_vm_end; extern vm_offset_t clean_sva, clean_eva; -extern int cpu_class; #define PV_FREELIST_MIN ((PAGE_SIZE / sizeof (struct pv_entry)) / 2) @@ -182,6 +188,14 @@ static caddr_t CADDR2; static pt_entry_t *msgbufmap; struct msgbuf *msgbufp=0; +#if defined(SMP) || defined(APIC_IO) +static pt_entry_t *apic_map; +#endif /* SMP || APIC_IO */ + +#if defined(APIC_IO) +static pt_entry_t *io_apic_map; +#endif /* APIC_IO */ + pt_entry_t *PMAP1 = 0; unsigned *PADDR1 = 0; @@ -297,6 +311,20 @@ pmap_bootstrap(firstaddr, loadaddr) va = virtual_avail; pte = (pt_entry_t *) pmap_pte(kernel_pmap, va); +#if defined(SMP) || defined(APIC_IO) + /* + * apic_map is the pt for where the local (CPU) apic is mapped in. + */ + SYSMAP(unsigned int *, apic_map, apic_base, 1) +#endif /* SMP || APIC_IO */ + +#if defined(APIC_IO) + /* + * io_apic_map is the pt for where the I/O apic is mapped in. + */ + SYSMAP(unsigned int *, io_apic_map, io_apic_base, 1) +#endif /* APIC_IO */ + /* * CMAP1/CMAP2 are used for zeroing and copying pages. */ @@ -323,14 +351,50 @@ pmap_bootstrap(firstaddr, loadaddr) virtual_avail = va; - *(int *) CMAP1 = *(int *) CMAP2 = *(int *) PTD = 0; + *(int *) CMAP1 = *(int *) CMAP2 = 0; + invltlb(); + +#if !defined(SMP) if (cpu_feature & CPUID_PGE) pgeflag = PG_G; else +#endif /* !SMP */ pgeflag = 0; +} +#if defined(SMP) || defined(APIC_IO) +void +pmap_bootstrap_apics() +{ + if (cpu_apic_address == 0) { + printf("pmap: BSP APIC address NOT found!\n"); + panic("pmap_bootstrap_apics: no apic"); + } + + *(int*)apic_map = PG_V | PG_RW | ((u_long)cpu_apic_address & PG_FRAME); + +#if defined(APIC_IO) +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX +#else + *(int*)io_apic_map = PG_V | PG_RW + | ((u_long)io_apic_address[0] & PG_FRAME); +#endif /* MULTIPLE_IOAPICS */ +#endif /* APIC_IO */ + + invltlb(); } +#endif /* SMP || APIC_IO */ + +#ifdef SMP +void +pmap_bootstrap2() +{ + *(int *) PTD = 0; + invltlb(); +} +#endif /* * Initialize the pmap module. diff --git a/sys/i386/i386/support.s b/sys/i386/i386/support.s index 0bfef221a9e1..5197f7ea1dc5 100644 --- a/sys/i386/i386/support.s +++ b/sys/i386/i386/support.s @@ -30,9 +30,10 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: support.s,v 1.50 1997/02/22 09:32:46 peter Exp $ + * $Id: support.s,v 1.51 1997/03/05 16:30:55 bde Exp $ */ +#include "opt_smp.h" #include "npx.h" #include "opt_cpu.h" @@ -40,10 +41,12 @@ #include <machine/cputypes.h> #include <machine/pmap.h> #include <machine/specialreg.h> +#include <machine/smpasm.h> #include "assym.s" #define KDSEL 0x10 /* kernel data selector */ +#define KCSEL 0x8 /* kernel code selector */ #define IDXSHIFT 10 .data @@ -193,7 +196,7 @@ do0: ret #endif -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_bzero) movl 4(%esp),%edx movl 8(%esp),%ecx @@ -436,7 +439,7 @@ ENTRY(generic_bcopy) cld ret -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_bcopy) pushl %esi pushl %edi @@ -621,7 +624,7 @@ ENTRY(copyout) jmp *_copyout_vector ENTRY(generic_copyout) - movl _curpcb,%eax + GETCURPCB(%eax) movl $copyout_fault,PCB_ONFAULT(%eax) pushl %esi pushl %edi @@ -714,7 +717,7 @@ ENTRY(generic_copyout) 3: movl %ebx,%ecx -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ALIGN_TEXT slow_copyout: #endif @@ -732,7 +735,7 @@ done_copyout: popl %edi popl %esi xorl %eax,%eax - movl _curpcb,%edx + GETCURPCB(%edx) movl %eax,PCB_ONFAULT(%edx) ret @@ -741,12 +744,12 @@ copyout_fault: popl %ebx popl %edi popl %esi - movl _curpcb,%edx + GETCURPCB(%edx) movl $0,PCB_ONFAULT(%edx) movl $EFAULT,%eax ret -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_copyout) /* * Duplicated from generic_copyout. Could be done a bit better. @@ -806,7 +809,7 @@ ENTRY(copyin) jmp *_copyin_vector ENTRY(generic_copyin) - movl _curpcb,%eax + GETCURPCB(%eax) movl $copyin_fault,PCB_ONFAULT(%eax) pushl %esi pushl %edi @@ -823,7 +826,7 @@ ENTRY(generic_copyin) cmpl $VM_MAXUSER_ADDRESS,%edx ja copyin_fault -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ALIGN_TEXT slow_copyin: #endif @@ -837,14 +840,14 @@ slow_copyin: rep movsb -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ALIGN_TEXT done_copyin: #endif popl %edi popl %esi xorl %eax,%eax - movl _curpcb,%edx + GETCURPCB(%edx) movl %eax,PCB_ONFAULT(%edx) ret @@ -852,12 +855,12 @@ done_copyin: copyin_fault: popl %edi popl %esi - movl _curpcb,%edx + GETCURPCB(%edx) movl $0,PCB_ONFAULT(%edx) movl $EFAULT,%eax ret -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) ENTRY(i586_copyin) /* * Duplicated from generic_copyin. Could be done a bit better. @@ -892,7 +895,7 @@ ENTRY(i586_copyin) jmp done_copyin #endif /* I586_CPU && NNPX > 0 */ -#if defined(I586_CPU) && NNPX > 0 +#if defined(I586_CPU) && NNPX > 0 && !defined(SMP) /* fastmove(src, dst, len) src in %esi dst in %edi @@ -1084,7 +1087,7 @@ fastmove_tail_fault: * fu{byte,sword,word} : fetch a byte (sword, word) from user memory */ ENTRY(fuword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx /* from */ @@ -1107,7 +1110,7 @@ ENTRY(fuswintr) ret ENTRY(fusword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1119,7 +1122,7 @@ ENTRY(fusword) ret ENTRY(fubyte) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1132,7 +1135,7 @@ ENTRY(fubyte) ALIGN_TEXT fusufault: - movl _curpcb,%ecx + GETCURPCB(%ecx) xorl %eax,%eax movl %eax,PCB_ONFAULT(%ecx) decl %eax @@ -1142,7 +1145,7 @@ fusufault: * su{byte,sword,word}: write a byte (word, longword) to user memory */ ENTRY(suword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1186,12 +1189,12 @@ ENTRY(suword) movl 8(%esp),%eax movl %eax,(%edx) xorl %eax,%eax - movl _curpcb,%ecx + GETCURPCB(%ecx) movl %eax,PCB_ONFAULT(%ecx) ret ENTRY(susword) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1235,13 +1238,13 @@ ENTRY(susword) movw 8(%esp),%ax movw %ax,(%edx) xorl %eax,%eax - movl _curpcb,%ecx /* restore trashed register */ + GETCURPCB(%ecx) /* restore trashed register */ movl %eax,PCB_ONFAULT(%ecx) ret ALTENTRY(suibyte) ENTRY(subyte) - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx @@ -1284,7 +1287,7 @@ ENTRY(subyte) movb 8(%esp),%al movb %al,(%edx) xorl %eax,%eax - movl _curpcb,%ecx /* restore trashed register */ + GETCURPCB(%ecx) /* restore trashed register */ movl %eax,PCB_ONFAULT(%ecx) ret @@ -1298,7 +1301,7 @@ ENTRY(subyte) ENTRY(copyinstr) pushl %esi pushl %edi - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $cpystrflt,PCB_ONFAULT(%ecx) movl 12(%esp),%esi /* %esi = from */ @@ -1346,7 +1349,7 @@ cpystrflt: cpystrflt_x: /* set *lencopied and return %eax */ - movl _curpcb,%ecx + GETCURPCB(%ecx) movl $0,PCB_ONFAULT(%ecx) movl 20(%esp),%ecx subl %edx,%ecx @@ -1446,13 +1449,14 @@ ENTRY(lgdt) movl $KDSEL,%eax movl %ax,%ds movl %ax,%es + movl %ax,%fs + movl %ax,%gs movl %ax,%ss /* reload code selector by turning return into intersegmental return */ movl (%esp),%eax pushl %eax -# movl $KCSEL,4(%esp) - movl $8,4(%esp) + movl $KCSEL,4(%esp) lret /* diff --git a/sys/i386/i386/swtch.s b/sys/i386/i386/swtch.s index a74cd09b5a6c..014ddd2f2c57 100644 --- a/sys/i386/i386/swtch.s +++ b/sys/i386/i386/swtch.s @@ -33,16 +33,24 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: swtch.s,v 1.46 1997/04/20 06:41:26 phk Exp $ + * $Id: swtch.s,v 1.47 1997/04/22 06:55:29 jdp Exp $ */ #include "npx.h" #include "opt_user_ldt.h" +#include "opt_smp.h" +#include "opt_smp_privpages.h" #include <sys/rtprio.h> #include <machine/asmacros.h> #include <machine/spl.h> +#include <machine/smpasm.h> +#include <machine/smptests.h> /** TEST_LOPRIO */ + +#if defined(SMP) && defined(SMP_PRIVPAGES) +#include <machine/pmap.h> +#endif #include "assym.s" @@ -61,8 +69,12 @@ * queues. */ .data +#ifndef SMP .globl _curpcb _curpcb: .long 0 /* pointer to curproc's PCB area */ +#endif + .globl _whichqs, _whichrtqs, _whichidqs + _whichqs: .long 0 /* which run queues have data */ _whichrtqs: .long 0 /* which realtime run queues have data */ _whichidqs: .long 0 /* which idletime run queues have data */ @@ -229,16 +241,30 @@ rem3id: .asciz "remrq.id" /* * When no processes are on the runq, cpu_switch() branches to _idle * to wait for something to come ready. + * + * NOTE: on an SMP system this routine is a startup-only code path. + * once initialization is over, meaning the idle procs have been + * created, we should NEVER branch here. */ ALIGN_TEXT _idle: +#ifdef SMP + movl _smp_active, %eax + cmpl $0, %eax + jnz badsw +#endif /* SMP */ xorl %ebp,%ebp movl $HIDENAME(tmpstk),%esp movl _IdlePTD,%ecx movl %ecx,%cr3 /* update common_tss.tss_esp0 pointer */ +#ifdef SMP + GETCPUID(%eax) + movl _SMPcommon_tss_ptr(,%eax,4), %eax +#else movl $_common_tss, %eax +#endif movl %esp, TSS_ESP0(%eax) #ifdef TSS_IS_CACHED /* example only */ @@ -288,13 +314,20 @@ ENTRY(default_halt) * cpu_switch() */ ENTRY(cpu_switch) + /* switch to new process. first, save context as needed */ - movl _curproc,%ecx + GETCURPROC(%ecx) /* if no process to save, don't bother */ testl %ecx,%ecx je sw1 +#ifdef SMP + movb P_ONCPU(%ecx), %al /* save "last" cpu */ + movb %al, P_LASTCPU(%ecx) + movb $0xff, P_ONCPU(%ecx) /* "leave" the cpu */ +#endif + movl P_ADDR(%ecx),%ecx movl (%esp),%eax /* Hardware registers */ @@ -305,10 +338,19 @@ ENTRY(cpu_switch) movl %esi,PCB_ESI(%ecx) movl %edi,PCB_EDI(%ecx) +#ifdef SMP + movl _mp_lock, %eax + cmpl $0xffffffff, %eax /* is it free? */ + je badsw /* yes, bad medicine! */ + andl $0x00ffffff, %eax /* clear CPU portion */ + movl %eax,PCB_MPNEST(%ecx) /* store it */ +#endif /* SMP */ + #if NNPX > 0 /* have we used fp, and need a save? */ - mov _curproc,%eax - cmp %eax,_npxproc + GETCURPROC(%eax) + GETNPXPROC(%ebx) + cmp %eax,%ebx jne 1f addl $PCB_SAVEFPU,%ecx /* h/w bugs make saving complicated */ pushl %ecx @@ -319,7 +361,7 @@ ENTRY(cpu_switch) movb $1,_intr_nesting_level /* charge Intr, not Sys/Idle */ - movl $0,_curproc /* out of process */ + SETCURPROC($0, %edi) /* save is done, now choose a new process or idle */ sw1: @@ -417,9 +459,30 @@ swtch_com: movl P_ADDR(%ecx),%edx movl PCB_CR3(%edx),%ebx +#if defined(SMP) && defined(SMP_PRIVPAGES) + /* Grab the private PT pointer from the outgoing process's PTD */ + movl $_PTD,%esi + movl 4*MPPTDI(%esi), %eax /* fetch cpu's prv pt */ +#endif + /* switch address space */ movl %ebx,%cr3 +#if defined(SMP) && defined(SMP_PRIVPAGES) + /* Copy the private PT to the new process's PTD */ + /* XXX yuck, the _PTD changes when we switch, so we have to + * reload %cr3 after changing the address space. + * We need to fix this by storing a pointer to the virtual + * location of the per-process PTD in the PCB or something quick. + * Dereferencing proc->vm_map->pmap->p_pdir[] is painful in asm. + */ + movl $_PTD,%esi + movl %eax, 4*MPPTDI(%esi) /* restore cpu's prv page */ + + /* XXX: we have just changed the page tables.. reload.. */ + movl %ebx,%cr3 +#endif + #ifdef HOW_TO_SWITCH_TSS /* example only */ /* Fix up tss pointer to floating pcb/stack structure */ /* XXX probably lots faster to store the 64 bits of tss entry @@ -435,7 +498,12 @@ swtch_com: #endif /* update common_tss.tss_esp0 pointer */ +#ifdef SMP + GETCPUID(%eax) + movl _SMPcommon_tss_ptr(,%eax,4), %eax +#else movl $_common_tss, %eax +#endif movl %edx, %ebx /* pcb */ addl $(UPAGES * PAGE_SIZE), %ebx movl %ebx, TSS_ESP0(%eax) @@ -457,9 +525,32 @@ swtch_com: movl PCB_EIP(%edx),%eax movl %eax,(%esp) - movl %edx,_curpcb - movl %ecx,_curproc /* into next process */ +#ifdef SMP + GETCPUID(%eax) + movb %al, P_ONCPU(%ecx) +#endif + SETCURPCB(%edx, %eax) + SETCURPROC(%ecx, %eax) + movb $0,_intr_nesting_level +#ifdef SMP + movl _apic_base, %eax /* base addr of LOCAL APIC */ +#if defined(TEST_LOPRIO) + /** + * FIXME: the belief here is that we ALWAYS leave here + * holding a the lock, is this TRUE??? + */ + pushl %edx + movl APIC_TPR(%eax), %edx /* get TPR register contents */ + andl $~0xff, %edx /* clear the prio field */ + movl %edx, APIC_TPR(%eax) /* now hold loprio for INTs */ + popl %edx +#endif /* TEST_LOPRIO */ + movl APIC_ID(%eax), %eax /* APIC ID register */ + andl $APIC_ID_MASK, %eax /* extract ID portion */ + orl PCB_MPNEST(%edx), %eax /* add count from PROC */ + movl %eax, _mp_lock /* load the mp_lock */ +#endif /* SMP */ #ifdef USER_LDT cmpl $0, PCB_USERLDT(%edx) @@ -520,7 +611,7 @@ ENTRY(savectx) * have to handle h/w bugs for reloading. We used to lose the * parent's npx state for forks by forgetting to reload. */ - mov _npxproc,%eax + GETNPXPROC(%eax) testl %eax,%eax je 1f diff --git a/sys/i386/i386/symbols.raw b/sys/i386/i386/symbols.raw index 2357c356525e..e530aacc7579 100644 --- a/sys/i386/i386/symbols.raw +++ b/sys/i386/i386/symbols.raw @@ -1,13 +1,12 @@ # @(#)symbols.raw 7.6 (Berkeley) 5/8/91 # -# $Id: symbols.raw,v 1.9 1997/04/07 07:15:55 peter Exp $ +# $Id: symbols.raw,v 1.10 1997/04/16 15:09:37 ache Exp $ # #gdb _IdlePTD _PTD - _curpcb _panicstr _atdevbase # _version diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c index 8a86965196d5..aa4e3c5d39d5 100644 --- a/sys/i386/i386/trap.c +++ b/sys/i386/i386/trap.c @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)trap.c 7.4 (Berkeley) 5/13/91 - * $Id: trap.c,v 1.91 1997/04/07 07:15:55 peter Exp $ + * $Id: trap.c,v 1.92 1997/04/14 13:52:52 bde Exp $ */ /* @@ -44,6 +44,7 @@ #include "opt_ktrace.h" #include "opt_ddb.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -76,6 +77,7 @@ #include <machine/reg.h> #include <machine/trap.h> #include <machine/../isa/isa_device.h> +#include <machine/smp.h> #ifdef POWERFAIL_NMI #include <sys/syslog.h> @@ -85,7 +87,11 @@ #include "isa.h" #include "npx.h" +#ifdef SMP +extern struct i386tss *SMPcommon_tss_ptr[NCPU]; +#else extern struct i386tss common_tss; +#endif int (*pmath_emulate) __P((struct trapframe *)); @@ -678,6 +684,9 @@ trap_fatal(frame) printf("\n\nFatal trap %d: %s while in %s mode\n", type, trap_msg[type], ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel"); +#ifdef SMP + printf("cpunumber = %d\n", cpunumber()); +#endif if (type == T_PAGEFLT) { printf("fault virtual address = 0x%x\n", eva); printf("fault code = %s %s, %s\n", @@ -740,6 +749,7 @@ trap_fatal(frame) if (kdb_trap (type, 0, frame)) return; #endif + printf("trap number = %d\n", type); if (type <= MAX_TRAP_MSG) panic(trap_msg[type]); else @@ -761,11 +771,20 @@ trap_fatal(frame) void dblfault_handler() { +#ifdef SMP + int x = cpunumber(); +#endif printf("\nFatal double fault:\n"); +#ifdef SMP + printf("eip = 0x%x\n", SMPcommon_tss_ptr[x]->tss_eip); + printf("esp = 0x%x\n", SMPcommon_tss_ptr[x]->tss_esp); + printf("ebp = 0x%x\n", SMPcommon_tss_ptr[x]->tss_ebp); +#else printf("eip = 0x%x\n", common_tss.tss_eip); printf("esp = 0x%x\n", common_tss.tss_esp); printf("ebp = 0x%x\n", common_tss.tss_ebp); +#endif panic("double fault"); } diff --git a/sys/i386/i386/tsc.c b/sys/i386/i386/tsc.c index b9425536d67a..d46f982806bd 100644 --- a/sys/i386/i386/tsc.c +++ b/sys/i386/i386/tsc.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)clock.c 7.2 (Berkeley) 5/12/91 - * $Id: clock.c,v 1.79 1997/03/05 08:08:48 bde Exp $ + * $Id: clock.c,v 1.80 1997/04/06 13:25:48 mckay Exp $ */ /* @@ -49,6 +49,7 @@ */ #include "opt_clock.h" +#include "opt_smp.h" #include "opt_cpu.h" #include <sys/param.h> @@ -100,7 +101,7 @@ int adjkerntz; /* local offset from GMT in seconds */ int disable_rtc_set; /* disable resettodr() if != 0 */ u_int idelayed; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) u_int i586_ctr_bias; u_int i586_ctr_comultiplier; u_int i586_ctr_freq; @@ -143,7 +144,7 @@ static u_char timer0_state; static u_char timer2_state; static void (*timer_func) __P((struct clockframe *frame)) = hardclock; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) static void set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq); #endif static void set_timer_freq(u_int freq, int intr_freq); @@ -553,7 +554,7 @@ calibrate_clocks(void) goto fail; tot_count = 0; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) if (cpu_class == CPUCLASS_586 || cpu_class == CPUCLASS_686) wrmsr(0x10, 0LL); /* XXX 0x10 is the MSR for the TSC */ #endif @@ -586,7 +587,7 @@ calibrate_clocks(void) goto fail; } -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * Read the cpu cycle counter. The timing considerations are * similar to those for the i8254 clock. @@ -667,14 +668,14 @@ startrtclock() printf( "%d Hz differs from default of %d Hz by more than 1%%\n", freq, timer_freq); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) i586_ctr_freq = 0; #endif } set_timer_freq(timer_freq, hz); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) #ifndef CLK_USE_I586_CALIBRATION if (i586_ctr_freq != 0) { if (bootverbose) @@ -821,6 +822,20 @@ resettodr() writertc(RTC_STATUSB, rtc_statusb); } +#if defined(APIC_IO) + +/* from icu.s: */ +extern u_int hwisrs[]; +extern void vec8254 __P((void)); +extern void vecRTC __P((void)); +extern u_int ivectors[]; +extern u_int Xintr8254; +extern u_int XintrRTC; +extern u_int mask8254; +extern u_int maskRTC; + +#endif /* APIC_IO */ + /* * Start both clocks running. */ @@ -828,6 +843,9 @@ void cpu_initclocks() { int diag; +#if defined(APIC_IO) + int x; +#endif /* APIC_IO */ if (statclock_disable) { /* @@ -845,11 +863,47 @@ cpu_initclocks() } /* Finish initializing 8253 timer 0. */ +#if defined(APIC_IO) + /* 8254 is traditionally on ISA IRQ0 */ + if ((x = get_isa_apic_irq(0)) < 0) { + /* + * bummer, this mb doesn't have the 8254 on ISA irq0, + * perhaps it's on the EISA bus... + */ + if ((x = get_eisa_apic_irq(0)) < 0) { + /* double bummer, attempt to redirect thru the 8259 */ + if (bootverbose) + printf("APIC missing 8254 connection\n"); + + /* allow 8254 timer to INTerrupt 8259 */ +#if !defined(IO_ICU1) +#define IO_ICU1 0x20 +#endif + x = inb(IO_ICU1 + 1); /* current mask in 8259 */ + x &= ~1; /* clear 8254 timer mask */ + outb(IO_ICU1 + 1, x); /* write new mask */ + + /* program IO APIC for type 3 INT on INT0 */ + if (ext_int_setup(0, 0) < 0) + panic("8254 redirect impossible!"); + x = 0; /* 8259 is on 0 */ + } + } + + hwisrs[x] = (u_int)vec8254; + Xintr8254 = (u_int)ivectors[x]; /* XXX might need Xfastintr# */ + mask8254 = (1 << x); + register_intr(/* irq */ x, /* XXX id */ 0, /* flags */ 0, + /* XXX */ (inthand2_t *)clkintr, &clk_imask, + /* unit */ 0); + INTREN(mask8254); +#else register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0, /* XXX */ (inthand2_t *)clkintr, &clk_imask, /* unit */ 0); INTREN(IRQ0); -#if defined(I586_CPU) || defined(I686_CPU) +#endif /* APIC_IO */ +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * Finish setting up anti-jitter measures. */ @@ -867,11 +921,36 @@ cpu_initclocks() diag = rtcin(RTC_DIAG); if (diag != 0) printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS); +#if defined(APIC_IO) + /* RTC is traditionally on ISA IRQ8 */ + if ((x = get_isa_apic_irq(8)) < 0) { + if ((x = get_eisa_apic_irq(8)) < 0) { + panic("APIC missing RTC connection"); + } + } + + hwisrs[x] = (u_int)vecRTC; + XintrRTC = (u_int)ivectors[x]; /* XXX might need Xfastintr# */ + maskRTC = (1 << x); + register_intr(/* irq */ x, /* XXX id */ 1, /* flags */ 0, + /* XXX */ (inthand2_t *)rtcintr, &stat_imask, + /* unit */ 0); + INTREN(maskRTC); +#else register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0, /* XXX */ (inthand2_t *)rtcintr, &stat_imask, /* unit */ 0); INTREN(IRQ8); +#endif /* APIC_IO */ writertc(RTC_STATUSB, rtc_statusb); + +#if defined(APIC_IO) + printf("Enabled INTs: "); + for (x = 0; x < 24; ++x) + if ((imen & (1 << x)) == 0) + printf("%d, ", x); + printf("imen: 0x%08x\n", imen); +#endif /* APIC_IO */ } void @@ -900,7 +979,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS if (timer0_state != 0) return (EBUSY); /* too much trouble to handle */ set_timer_freq(freq, hz); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) set_i586_ctr_freq(i586_ctr_freq, timer_freq); #endif } @@ -910,7 +989,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS SYSCTL_PROC(_machdep, OID_AUTO, i8254_freq, CTLTYPE_INT | CTLFLAG_RW, 0, sizeof(u_int), sysctl_machdep_i8254_freq, "I", ""); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) static void set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq) { @@ -949,4 +1028,4 @@ sysctl_machdep_i586_freq SYSCTL_HANDLER_ARGS SYSCTL_PROC(_machdep, OID_AUTO, i586_freq, CTLTYPE_INT | CTLFLAG_RW, 0, sizeof(u_int), sysctl_machdep_i586_freq, "I", ""); -#endif /* defined(I586_CPU) || defined(I686_CPU) */ +#endif /* (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) */ diff --git a/sys/i386/include/apic.h b/sys/i386/include/apic.h new file mode 100644 index 000000000000..a585efba626d --- /dev/null +++ b/sys/i386/include/apic.h @@ -0,0 +1,409 @@ +/* + * Copyright (c) 1996, by Peter Wemm and Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: apic.h,v 1.17 1997/04/26 06:41:36 peter Exp $ + */ + +#ifndef _MACHINE_APIC_H_ +#define _MACHINE_APIC_H_ + +/* + * Local && I/O APIC definitions. + */ + +/* + * Pentium P54C+ Build-in APIC + * (Advanced programmable Interrupt Controller) + * + * Base Address of Build-in APIC in memory location + * is 0xfee00000. + * + * Map of APIC REgisters: + * + * Offset (hex) Description Read/Write state + * 000 Reserved + * 010 Reserved + * 020 ID Local APIC ID R/W + * 030 VER Local APIC Version R + * 040 Reserved + * 050 Reserved + * 060 Reserved + * 070 Reserved + * 080 Task Priority Register R/W + * 090 Arbitration Priority Register R + * 0A0 Processor Priority Register R + * 0B0 EOI Register W + * 0C0 RRR Remote read R + * 0D0 Logical Destination R/W + * 0E0 Destination Format Register 0..27 R; 28..31 R/W + * 0F0 SVR Spurious Interrupt Vector Reg. 0..3 R; 4..9 R/W + * 100 ISR 000-031 R + * 110 ISR 032-063 R + * 120 ISR 064-095 R + * 130 ISR 095-128 R + * 140 ISR 128-159 R + * 150 ISR 160-191 R + * 160 ISR 192-223 R + * 170 ISR 224-255 R + * 180 TMR 000-031 R + * 190 TMR 032-063 R + * 1A0 TMR 064-095 R + * 1B0 TMR 095-128 R + * 1C0 TMR 128-159 R + * 1D0 TMR 160-191 R + * 1E0 TMR 192-223 R + * 1F0 TMR 224-255 R + * 200 IRR 000-031 R + * 210 IRR 032-063 R + * 220 IRR 064-095 R + * 230 IRR 095-128 R + * 240 IRR 128-159 R + * 250 IRR 160-191 R + * 260 IRR 192-223 R + * 270 IRR 224-255 R + * 280 Error Status Register R + * 290 Reserved + * 2A0 Reserved + * 2B0 Reserved + * 2C0 Reserved + * 2D0 Reserved + * 2E0 Reserved + * 2F0 Reserved + * 300 ICR_LOW Interrupt Command Reg. (0-31) R/W + * 310 ICR_HI Interrupt Command Reg. (32-63) R/W + * 320 Local Vector Table (Timer) R/W + * 330 Reserved + * 340 Reserved + * 350 LVT1 Local Vector Table (LINT0) R/W + * 360 LVT2 Local Vector Table (LINT1) R/W + * 370 LVT3 Local Vector Table (ERROR) R/W + * 380 Initial Count Reg. for Timer R/W + * 390 Current Count of Timer R + * 3A0 Reserved + * 3B0 Reserved + * 3C0 Reserved + * 3D0 Reserved + * 3E0 Timer Divide Configuration Reg. R/W + * 3F0 Reserved + */ + + +/****************************************************************************** + * global defines, etc. + */ + +/* enable the InterProcessor Interrupt code, FIXME: temporary marker */ +#define IPI_INTS + +/** + * this enables code concerned with handling more than one IO APIC. + * Note: this is NOT READY for use! + * +#define MULTIPLE_IOAPICS + */ + +/****************************************************************************** + * LOCAL APIC defines + */ + +/* default physical locations of LOCAL (CPU) APICs */ +#define DEFAULT_APIC_BASE 0xfee00000 + +# if defined(LOCORE) + +#define APIC_ID 0x020 +#define APIC_VER 0x030 +#define APIC_TPR 0x080 +#define APIC_APR 0x090 +#define APIC_PPR 0x0a0 +#define APIC_EOI 0x0b0 +#define APIC_RR 0x0c0 +#define APIC_LDR 0x0d0 +#define APIC_DFR 0x0e0 +#define APIC_SVR 0x0f0 +#define APIC_ISR 0x100 +#define APIC_ISR0 0x100 +#define APIC_ISR1 0x110 +#define APIC_ISR2 0x120 +#define APIC_TMR 0x180 +#define APIC_IRR 0x200 +#define APIC_IRR0 0x200 +#define APIC_IRR1 0x210 +#define APIC_IRR2 0x220 +#define APIC_ESR 0x280 +#define APIC_ICR_LOW 0x300 +#define APIC_ICR_HI 0x310 +#define APIC_LVTT 0x320 +#define APIC_LVT1 0x350 +#define APIC_LVT2 0x360 +#define APIC_LVT3 0x370 +#define APIC_TICR 0x380 +#define APIC_TCCR 0x390 +#define APIC_TDCR 0x3e0 + +# else /* !LOCORE */ + +/* offsets in apic_base[] */ +#define APIC_ID (0x020/4) +#define APIC_VER (0x030/4) +#define APIC_TPR (0x080/4) +#define APIC_APR (0x090/4) +#define APIC_PPR (0x0a0/4) +#define APIC_EOI (0x0b0/4) +#define APIC_RR (0x0c0/4) +#define APIC_LDR (0x0d0/4) +#define APIC_DFR (0x0e0/4) +#define APIC_SVR (0x0f0/4) +#define APIC_ISR (0x100/4) +#define APIC_ISR0 (0x100/4) +#define APIC_ISR1 (0x110/4) +#define APIC_ISR2 (0x120/4) +#define APIC_TMR (0x180/4) +#define APIC_IRR (0x200/4) +#define APIC_IRR0 (0x200/4) +#define APIC_IRR1 (0x210/4) +#define APIC_IRR2 (0x220/4) +#define APIC_ESR (0x280/4) +#define APIC_ICR_LOW (0x300/4) +#define APIC_ICR_HI (0x310/4) +#define APIC_LVTT (0x320/4) +#define APIC_LVT1 (0x350/4) +#define APIC_LVT2 (0x360/4) +#define APIC_LVT3 (0x370/4) +#define APIC_TICR (0x380/4) +#define APIC_TCCR (0x390/4) +#define APIC_TDCR (0x3e0/4) + +# endif /* LOCORE */ + + +/* fields in VER */ +#define APIC_VER_VERSION 0x000000ff +#define APIC_VER_MAXLVT 0x00ff0000 +#define MAXLVTSHIFT 16 + +/* fields in SVR */ +#define APIC_SVR_ENABLE 0x00000100 +# define APIC_SVR_SWDIS 0x00000000 +# define APIC_SVR_SWEN 0x00000100 +#define APIC_SVR_FOCUS 0x00000200 +# define APIC_SVR_FEN 0x00000000 +# define APIC_SVR_FDIS 0x00000200 +#define APIC_TPR_PRIO 0x000000ff +# define APIC_TPR_INT 0x000000f0 +# define APIC_TPR_SUB 0x0000000f + + +/* fields in ICR_LOW */ +#define APIC_VECTOR_MASK 0x000000ff + +#define APIC_DELMODE_MASK 0x00000700 +# define APIC_DELMODE_FIXED 0x00000000 +# define APIC_DELMODE_LOWPRIO 0x00000100 +# define APIC_DELMODE_SMI 0x00000200 +# define APIC_DELMODE_RR 0x00000300 +# define APIC_DELMODE_NMI 0x00000400 +# define APIC_DELMODE_INIT 0x00000500 +# define APIC_DELMODE_STARTUP 0x00000600 +# define APIC_DELMODE_RESV 0x00000700 + +#define APIC_DESTMODE_MASK 0x00000800 +# define APIC_DESTMODE_PHY 0x00000000 +# define APIC_DESTMODE_LOG 0x00000800 + +#define APIC_DELSTAT_MASK 0x00001000 +# define APIC_DELSTAT_IDLE 0x00000000 +# define APIC_DELSTAT_PEND 0x00001000 + +#define APIC_RESV1_MASK 0x00002000 + +#define APIC_LEVEL_MASK 0x00004000 +# define APIC_LEVEL_DEASSERT 0x00000000 +# define APIC_LEVEL_ASSERT 0x00004000 + +#define APIC_TRIGMOD_MASK 0x00008000 +# define APIC_TRIGMOD_EDGE 0x00000000 +# define APIC_TRIGMOD_LEVEL 0x00008000 + +#define APIC_RRSTAT_MASK 0x00030000 +# define APIC_RRSTAT_INVALID 0x00000000 +# define APIC_RRSTAT_INPROG 0x00010000 +# define APIC_RRSTAT_VALID 0x00020000 +# define APIC_RRSTAT_RESV 0x00030000 + +#define APIC_DEST_MASK 0x000c0000 +# define APIC_DEST_DESTFLD 0x00000000 +# define APIC_DEST_SELF 0x00040000 +# define APIC_DEST_ALLISELF 0x00080000 +# define APIC_DEST_ALLESELF 0x000c0000 + +#define APIC_RESV2_MASK 0xfff00000 + + +/* fields in ICR_HIGH */ +#define APIC_ID_MASK 0x0f000000 + + +/* fields in LVT1/2 */ +#define APIC_LVT_VECTOR 0x000000ff +#define APIC_LVT_DS 0x00001000 +#define APIC_LVT_M 0x00010000 + + +/* fields in LVT Timer */ +#define APIC_LVTT_VECTOR 0x000000ff +#define APIC_LVTT_DS 0x00001000 +#define APIC_LVTT_M 0x00010000 +#define APIC_LVTT_TM 0x00020000 + + +/* fields in TDCR */ +#define APIC_TDCR_2 0x00 +#define APIC_TDCR_4 0x01 +#define APIC_TDCR_8 0x02 +#define APIC_TDCR_16 0x03 +#define APIC_TDCR_32 0x08 +#define APIC_TDCR_64 0x09 +#define APIC_TDCR_128 0x0a +#define APIC_TDCR_1 0x0b + + +/* + * fields in IRR + * ISA INTerrupts are in bits 16-31 of the 1st IRR register. + * these masks DON'T EQUAL the isa IRQs of the same name. + * FIXME: how do we make this portable for MP table configurations??? + * look for "HARD_VECTORXXX" marking places with this problem. + */ +#define APIC_IRQ0 0x00000001 +#define APIC_IRQ1 0x00000002 +#define APIC_IRQ2 0x00000004 +#define APIC_IRQ3 0x00000008 +#define APIC_IRQ4 0x00000010 +#define APIC_IRQ5 0x00000020 +#define APIC_IRQ6 0x00000040 +#define APIC_IRQ7 0x00000080 +#define APIC_IRQ8 0x00000100 +#define APIC_IRQ9 0x00000200 +#define APIC_IRQ10 0x00000400 +#define APIC_IRQ11 0x00000800 +#define APIC_IRQ12 0x00001000 +#define APIC_IRQ13 0x00002000 +#define APIC_IRQ14 0x00004000 +#define APIC_IRQ15 0x00008000 +#define APIC_IRQ16 0x00010000 +#define APIC_IRQ17 0x00020000 +#define APIC_IRQ18 0x00040000 +#define APIC_IRQ19 0x00080000 +#define APIC_IRQ20 0x00100000 +#define APIC_IRQ21 0x00200000 +#define APIC_IRQ22 0x00400000 +#define APIC_IRQ23 0x00800000 + + +/****************************************************************************** + * I/O APIC defines + */ + +/* default physical locations of an IO APIC */ +#define DEFAULT_IO_APIC_BASE 0xfec00000 + +/* window register offset */ +#define IOAPIC_WINDOW 0x10 + +/* indexes into IO APIC */ +#define IOAPIC_ID 0x00 +#define IOAPIC_VER 0x01 +#define IOAPIC_ARB 0x02 +#define IOAPIC_REDTBL 0x10 +#define IOAPIC_REDTBL0 IOAPIC_REDTBL +#define IOAPIC_REDTBL1 (IOAPIC_REDTBL+0x02) +#define IOAPIC_REDTBL2 (IOAPIC_REDTBL+0x04) +#define IOAPIC_REDTBL3 (IOAPIC_REDTBL+0x06) +#define IOAPIC_REDTBL4 (IOAPIC_REDTBL+0x08) +#define IOAPIC_REDTBL5 (IOAPIC_REDTBL+0x0a) +#define IOAPIC_REDTBL6 (IOAPIC_REDTBL+0x0c) +#define IOAPIC_REDTBL7 (IOAPIC_REDTBL+0x0e) +#define IOAPIC_REDTBL8 (IOAPIC_REDTBL+0x10) +#define IOAPIC_REDTBL9 (IOAPIC_REDTBL+0x12) +#define IOAPIC_REDTBL10 (IOAPIC_REDTBL+0x14) +#define IOAPIC_REDTBL11 (IOAPIC_REDTBL+0x16) +#define IOAPIC_REDTBL12 (IOAPIC_REDTBL+0x18) +#define IOAPIC_REDTBL13 (IOAPIC_REDTBL+0x1a) +#define IOAPIC_REDTBL14 (IOAPIC_REDTBL+0x1c) +#define IOAPIC_REDTBL15 (IOAPIC_REDTBL+0x1e) +#define IOAPIC_REDTBL16 (IOAPIC_REDTBL+0x20) +#define IOAPIC_REDTBL17 (IOAPIC_REDTBL+0x22) +#define IOAPIC_REDTBL18 (IOAPIC_REDTBL+0x24) +#define IOAPIC_REDTBL19 (IOAPIC_REDTBL+0x26) +#define IOAPIC_REDTBL20 (IOAPIC_REDTBL+0x28) +#define IOAPIC_REDTBL21 (IOAPIC_REDTBL+0x2a) +#define IOAPIC_REDTBL22 (IOAPIC_REDTBL+0x2c) +#define IOAPIC_REDTBL23 (IOAPIC_REDTBL+0x2e) + +/* fields in VER */ +#define IOART_VER_VERSION 0x000000ff +#define IOART_VER_MAXREDIR 0x00ff0000 +#define MAXREDIRSHIFT 16 + +/* + * fields in the IO APIC's redirection table entries + */ +#define IOART_DEST APIC_ID_MASK /* broadcast addr: all APICs */ + +#define IOART_RESV 0x00fe0000 /* reserved */ + +#define IOART_INTMASK 0x00010000 /* R/W: INTerrupt mask */ +# define IOART_INTMCLR 0x00000000 /* clear, allow INTs */ +# define IOART_INTMSET 0x00010000 /* set, inhibit INTs */ + +#define IOART_TRGRMOD 0x00008000 /* R/W: trigger mode */ +# define IOART_TRGREDG 0x00000000 /* edge */ +# define IOART_TRGRLVL 0x00008000 /* level */ + +#define IOART_REM_IRR 0x00004000 /* RO: remote IRR */ + +#define IOART_INTPOL 0x00002000 /* R/W: INT input pin polarity */ +# define IOART_INTAHI 0x00000000 /* active high */ +# define IOART_INTALO 0x00002000 /* active low */ + +#define IOART_DELIVS 0x00001000 /* RO: delivery status */ + +#define IOART_DESTMOD 0x00000800 /* R/W: destination mode */ +# define IOART_DESTPHY 0x00000000 /* physical */ +# define IOART_DESTLOG 0x00000800 /* logical */ + +#define IOART_DELMOD 0x00000700 /* R/W: delivery mode */ +# define IOART_DELFIXED 0x00000000 /* fixed */ +# define IOART_DELLOPRI 0x00000100 /* lowest priority */ +# define IOART_DELSMI 0x00000200 /* System Management INT */ +# define IOART_DELRSV1 0x00000300 /* reserved */ +# define IOART_DELNMI 0x00000400 /* NMI signal */ +# define IOART_DELINIT 0x00000500 /* INIT signal */ +# define IOART_DELRSV2 0x00000600 /* reserved */ +# define IOART_DELEXINT 0x00000700 /* External INTerrupt */ + +#define IOART_INTVEC 0x000000ff /* R/W: INTerrupt vector field */ + +#endif /* _MACHINE_APIC_H_ */ diff --git a/sys/i386/include/apicreg.h b/sys/i386/include/apicreg.h new file mode 100644 index 000000000000..a585efba626d --- /dev/null +++ b/sys/i386/include/apicreg.h @@ -0,0 +1,409 @@ +/* + * Copyright (c) 1996, by Peter Wemm and Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: apic.h,v 1.17 1997/04/26 06:41:36 peter Exp $ + */ + +#ifndef _MACHINE_APIC_H_ +#define _MACHINE_APIC_H_ + +/* + * Local && I/O APIC definitions. + */ + +/* + * Pentium P54C+ Build-in APIC + * (Advanced programmable Interrupt Controller) + * + * Base Address of Build-in APIC in memory location + * is 0xfee00000. + * + * Map of APIC REgisters: + * + * Offset (hex) Description Read/Write state + * 000 Reserved + * 010 Reserved + * 020 ID Local APIC ID R/W + * 030 VER Local APIC Version R + * 040 Reserved + * 050 Reserved + * 060 Reserved + * 070 Reserved + * 080 Task Priority Register R/W + * 090 Arbitration Priority Register R + * 0A0 Processor Priority Register R + * 0B0 EOI Register W + * 0C0 RRR Remote read R + * 0D0 Logical Destination R/W + * 0E0 Destination Format Register 0..27 R; 28..31 R/W + * 0F0 SVR Spurious Interrupt Vector Reg. 0..3 R; 4..9 R/W + * 100 ISR 000-031 R + * 110 ISR 032-063 R + * 120 ISR 064-095 R + * 130 ISR 095-128 R + * 140 ISR 128-159 R + * 150 ISR 160-191 R + * 160 ISR 192-223 R + * 170 ISR 224-255 R + * 180 TMR 000-031 R + * 190 TMR 032-063 R + * 1A0 TMR 064-095 R + * 1B0 TMR 095-128 R + * 1C0 TMR 128-159 R + * 1D0 TMR 160-191 R + * 1E0 TMR 192-223 R + * 1F0 TMR 224-255 R + * 200 IRR 000-031 R + * 210 IRR 032-063 R + * 220 IRR 064-095 R + * 230 IRR 095-128 R + * 240 IRR 128-159 R + * 250 IRR 160-191 R + * 260 IRR 192-223 R + * 270 IRR 224-255 R + * 280 Error Status Register R + * 290 Reserved + * 2A0 Reserved + * 2B0 Reserved + * 2C0 Reserved + * 2D0 Reserved + * 2E0 Reserved + * 2F0 Reserved + * 300 ICR_LOW Interrupt Command Reg. (0-31) R/W + * 310 ICR_HI Interrupt Command Reg. (32-63) R/W + * 320 Local Vector Table (Timer) R/W + * 330 Reserved + * 340 Reserved + * 350 LVT1 Local Vector Table (LINT0) R/W + * 360 LVT2 Local Vector Table (LINT1) R/W + * 370 LVT3 Local Vector Table (ERROR) R/W + * 380 Initial Count Reg. for Timer R/W + * 390 Current Count of Timer R + * 3A0 Reserved + * 3B0 Reserved + * 3C0 Reserved + * 3D0 Reserved + * 3E0 Timer Divide Configuration Reg. R/W + * 3F0 Reserved + */ + + +/****************************************************************************** + * global defines, etc. + */ + +/* enable the InterProcessor Interrupt code, FIXME: temporary marker */ +#define IPI_INTS + +/** + * this enables code concerned with handling more than one IO APIC. + * Note: this is NOT READY for use! + * +#define MULTIPLE_IOAPICS + */ + +/****************************************************************************** + * LOCAL APIC defines + */ + +/* default physical locations of LOCAL (CPU) APICs */ +#define DEFAULT_APIC_BASE 0xfee00000 + +# if defined(LOCORE) + +#define APIC_ID 0x020 +#define APIC_VER 0x030 +#define APIC_TPR 0x080 +#define APIC_APR 0x090 +#define APIC_PPR 0x0a0 +#define APIC_EOI 0x0b0 +#define APIC_RR 0x0c0 +#define APIC_LDR 0x0d0 +#define APIC_DFR 0x0e0 +#define APIC_SVR 0x0f0 +#define APIC_ISR 0x100 +#define APIC_ISR0 0x100 +#define APIC_ISR1 0x110 +#define APIC_ISR2 0x120 +#define APIC_TMR 0x180 +#define APIC_IRR 0x200 +#define APIC_IRR0 0x200 +#define APIC_IRR1 0x210 +#define APIC_IRR2 0x220 +#define APIC_ESR 0x280 +#define APIC_ICR_LOW 0x300 +#define APIC_ICR_HI 0x310 +#define APIC_LVTT 0x320 +#define APIC_LVT1 0x350 +#define APIC_LVT2 0x360 +#define APIC_LVT3 0x370 +#define APIC_TICR 0x380 +#define APIC_TCCR 0x390 +#define APIC_TDCR 0x3e0 + +# else /* !LOCORE */ + +/* offsets in apic_base[] */ +#define APIC_ID (0x020/4) +#define APIC_VER (0x030/4) +#define APIC_TPR (0x080/4) +#define APIC_APR (0x090/4) +#define APIC_PPR (0x0a0/4) +#define APIC_EOI (0x0b0/4) +#define APIC_RR (0x0c0/4) +#define APIC_LDR (0x0d0/4) +#define APIC_DFR (0x0e0/4) +#define APIC_SVR (0x0f0/4) +#define APIC_ISR (0x100/4) +#define APIC_ISR0 (0x100/4) +#define APIC_ISR1 (0x110/4) +#define APIC_ISR2 (0x120/4) +#define APIC_TMR (0x180/4) +#define APIC_IRR (0x200/4) +#define APIC_IRR0 (0x200/4) +#define APIC_IRR1 (0x210/4) +#define APIC_IRR2 (0x220/4) +#define APIC_ESR (0x280/4) +#define APIC_ICR_LOW (0x300/4) +#define APIC_ICR_HI (0x310/4) +#define APIC_LVTT (0x320/4) +#define APIC_LVT1 (0x350/4) +#define APIC_LVT2 (0x360/4) +#define APIC_LVT3 (0x370/4) +#define APIC_TICR (0x380/4) +#define APIC_TCCR (0x390/4) +#define APIC_TDCR (0x3e0/4) + +# endif /* LOCORE */ + + +/* fields in VER */ +#define APIC_VER_VERSION 0x000000ff +#define APIC_VER_MAXLVT 0x00ff0000 +#define MAXLVTSHIFT 16 + +/* fields in SVR */ +#define APIC_SVR_ENABLE 0x00000100 +# define APIC_SVR_SWDIS 0x00000000 +# define APIC_SVR_SWEN 0x00000100 +#define APIC_SVR_FOCUS 0x00000200 +# define APIC_SVR_FEN 0x00000000 +# define APIC_SVR_FDIS 0x00000200 +#define APIC_TPR_PRIO 0x000000ff +# define APIC_TPR_INT 0x000000f0 +# define APIC_TPR_SUB 0x0000000f + + +/* fields in ICR_LOW */ +#define APIC_VECTOR_MASK 0x000000ff + +#define APIC_DELMODE_MASK 0x00000700 +# define APIC_DELMODE_FIXED 0x00000000 +# define APIC_DELMODE_LOWPRIO 0x00000100 +# define APIC_DELMODE_SMI 0x00000200 +# define APIC_DELMODE_RR 0x00000300 +# define APIC_DELMODE_NMI 0x00000400 +# define APIC_DELMODE_INIT 0x00000500 +# define APIC_DELMODE_STARTUP 0x00000600 +# define APIC_DELMODE_RESV 0x00000700 + +#define APIC_DESTMODE_MASK 0x00000800 +# define APIC_DESTMODE_PHY 0x00000000 +# define APIC_DESTMODE_LOG 0x00000800 + +#define APIC_DELSTAT_MASK 0x00001000 +# define APIC_DELSTAT_IDLE 0x00000000 +# define APIC_DELSTAT_PEND 0x00001000 + +#define APIC_RESV1_MASK 0x00002000 + +#define APIC_LEVEL_MASK 0x00004000 +# define APIC_LEVEL_DEASSERT 0x00000000 +# define APIC_LEVEL_ASSERT 0x00004000 + +#define APIC_TRIGMOD_MASK 0x00008000 +# define APIC_TRIGMOD_EDGE 0x00000000 +# define APIC_TRIGMOD_LEVEL 0x00008000 + +#define APIC_RRSTAT_MASK 0x00030000 +# define APIC_RRSTAT_INVALID 0x00000000 +# define APIC_RRSTAT_INPROG 0x00010000 +# define APIC_RRSTAT_VALID 0x00020000 +# define APIC_RRSTAT_RESV 0x00030000 + +#define APIC_DEST_MASK 0x000c0000 +# define APIC_DEST_DESTFLD 0x00000000 +# define APIC_DEST_SELF 0x00040000 +# define APIC_DEST_ALLISELF 0x00080000 +# define APIC_DEST_ALLESELF 0x000c0000 + +#define APIC_RESV2_MASK 0xfff00000 + + +/* fields in ICR_HIGH */ +#define APIC_ID_MASK 0x0f000000 + + +/* fields in LVT1/2 */ +#define APIC_LVT_VECTOR 0x000000ff +#define APIC_LVT_DS 0x00001000 +#define APIC_LVT_M 0x00010000 + + +/* fields in LVT Timer */ +#define APIC_LVTT_VECTOR 0x000000ff +#define APIC_LVTT_DS 0x00001000 +#define APIC_LVTT_M 0x00010000 +#define APIC_LVTT_TM 0x00020000 + + +/* fields in TDCR */ +#define APIC_TDCR_2 0x00 +#define APIC_TDCR_4 0x01 +#define APIC_TDCR_8 0x02 +#define APIC_TDCR_16 0x03 +#define APIC_TDCR_32 0x08 +#define APIC_TDCR_64 0x09 +#define APIC_TDCR_128 0x0a +#define APIC_TDCR_1 0x0b + + +/* + * fields in IRR + * ISA INTerrupts are in bits 16-31 of the 1st IRR register. + * these masks DON'T EQUAL the isa IRQs of the same name. + * FIXME: how do we make this portable for MP table configurations??? + * look for "HARD_VECTORXXX" marking places with this problem. + */ +#define APIC_IRQ0 0x00000001 +#define APIC_IRQ1 0x00000002 +#define APIC_IRQ2 0x00000004 +#define APIC_IRQ3 0x00000008 +#define APIC_IRQ4 0x00000010 +#define APIC_IRQ5 0x00000020 +#define APIC_IRQ6 0x00000040 +#define APIC_IRQ7 0x00000080 +#define APIC_IRQ8 0x00000100 +#define APIC_IRQ9 0x00000200 +#define APIC_IRQ10 0x00000400 +#define APIC_IRQ11 0x00000800 +#define APIC_IRQ12 0x00001000 +#define APIC_IRQ13 0x00002000 +#define APIC_IRQ14 0x00004000 +#define APIC_IRQ15 0x00008000 +#define APIC_IRQ16 0x00010000 +#define APIC_IRQ17 0x00020000 +#define APIC_IRQ18 0x00040000 +#define APIC_IRQ19 0x00080000 +#define APIC_IRQ20 0x00100000 +#define APIC_IRQ21 0x00200000 +#define APIC_IRQ22 0x00400000 +#define APIC_IRQ23 0x00800000 + + +/****************************************************************************** + * I/O APIC defines + */ + +/* default physical locations of an IO APIC */ +#define DEFAULT_IO_APIC_BASE 0xfec00000 + +/* window register offset */ +#define IOAPIC_WINDOW 0x10 + +/* indexes into IO APIC */ +#define IOAPIC_ID 0x00 +#define IOAPIC_VER 0x01 +#define IOAPIC_ARB 0x02 +#define IOAPIC_REDTBL 0x10 +#define IOAPIC_REDTBL0 IOAPIC_REDTBL +#define IOAPIC_REDTBL1 (IOAPIC_REDTBL+0x02) +#define IOAPIC_REDTBL2 (IOAPIC_REDTBL+0x04) +#define IOAPIC_REDTBL3 (IOAPIC_REDTBL+0x06) +#define IOAPIC_REDTBL4 (IOAPIC_REDTBL+0x08) +#define IOAPIC_REDTBL5 (IOAPIC_REDTBL+0x0a) +#define IOAPIC_REDTBL6 (IOAPIC_REDTBL+0x0c) +#define IOAPIC_REDTBL7 (IOAPIC_REDTBL+0x0e) +#define IOAPIC_REDTBL8 (IOAPIC_REDTBL+0x10) +#define IOAPIC_REDTBL9 (IOAPIC_REDTBL+0x12) +#define IOAPIC_REDTBL10 (IOAPIC_REDTBL+0x14) +#define IOAPIC_REDTBL11 (IOAPIC_REDTBL+0x16) +#define IOAPIC_REDTBL12 (IOAPIC_REDTBL+0x18) +#define IOAPIC_REDTBL13 (IOAPIC_REDTBL+0x1a) +#define IOAPIC_REDTBL14 (IOAPIC_REDTBL+0x1c) +#define IOAPIC_REDTBL15 (IOAPIC_REDTBL+0x1e) +#define IOAPIC_REDTBL16 (IOAPIC_REDTBL+0x20) +#define IOAPIC_REDTBL17 (IOAPIC_REDTBL+0x22) +#define IOAPIC_REDTBL18 (IOAPIC_REDTBL+0x24) +#define IOAPIC_REDTBL19 (IOAPIC_REDTBL+0x26) +#define IOAPIC_REDTBL20 (IOAPIC_REDTBL+0x28) +#define IOAPIC_REDTBL21 (IOAPIC_REDTBL+0x2a) +#define IOAPIC_REDTBL22 (IOAPIC_REDTBL+0x2c) +#define IOAPIC_REDTBL23 (IOAPIC_REDTBL+0x2e) + +/* fields in VER */ +#define IOART_VER_VERSION 0x000000ff +#define IOART_VER_MAXREDIR 0x00ff0000 +#define MAXREDIRSHIFT 16 + +/* + * fields in the IO APIC's redirection table entries + */ +#define IOART_DEST APIC_ID_MASK /* broadcast addr: all APICs */ + +#define IOART_RESV 0x00fe0000 /* reserved */ + +#define IOART_INTMASK 0x00010000 /* R/W: INTerrupt mask */ +# define IOART_INTMCLR 0x00000000 /* clear, allow INTs */ +# define IOART_INTMSET 0x00010000 /* set, inhibit INTs */ + +#define IOART_TRGRMOD 0x00008000 /* R/W: trigger mode */ +# define IOART_TRGREDG 0x00000000 /* edge */ +# define IOART_TRGRLVL 0x00008000 /* level */ + +#define IOART_REM_IRR 0x00004000 /* RO: remote IRR */ + +#define IOART_INTPOL 0x00002000 /* R/W: INT input pin polarity */ +# define IOART_INTAHI 0x00000000 /* active high */ +# define IOART_INTALO 0x00002000 /* active low */ + +#define IOART_DELIVS 0x00001000 /* RO: delivery status */ + +#define IOART_DESTMOD 0x00000800 /* R/W: destination mode */ +# define IOART_DESTPHY 0x00000000 /* physical */ +# define IOART_DESTLOG 0x00000800 /* logical */ + +#define IOART_DELMOD 0x00000700 /* R/W: delivery mode */ +# define IOART_DELFIXED 0x00000000 /* fixed */ +# define IOART_DELLOPRI 0x00000100 /* lowest priority */ +# define IOART_DELSMI 0x00000200 /* System Management INT */ +# define IOART_DELRSV1 0x00000300 /* reserved */ +# define IOART_DELNMI 0x00000400 /* NMI signal */ +# define IOART_DELINIT 0x00000500 /* INIT signal */ +# define IOART_DELRSV2 0x00000600 /* reserved */ +# define IOART_DELEXINT 0x00000700 /* External INTerrupt */ + +#define IOART_INTVEC 0x000000ff /* R/W: INTerrupt vector field */ + +#endif /* _MACHINE_APIC_H_ */ diff --git a/sys/i386/include/clock.h b/sys/i386/include/clock.h index 30ff97a7f80f..df8a20c5e4e8 100644 --- a/sys/i386/include/clock.h +++ b/sys/i386/include/clock.h @@ -3,13 +3,13 @@ * Garrett Wollman, September 1994. * This file is in the public domain. * - * $Id$ + * $Id: clock.h,v 1.24 1997/02/22 09:33:59 peter Exp $ */ #ifndef _MACHINE_CLOCK_H_ #define _MACHINE_CLOCK_H_ -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) #define CPU_CLOCKUPDATE(otime, ntime) cpu_clockupdate((otime), (ntime)) #else #define CPU_CLOCKUPDATE(otime, ntime) (*(otime) = *(ntime)) @@ -28,7 +28,7 @@ */ extern int adjkerntz; extern int disable_rtc_set; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) extern u_int i586_ctr_bias; extern u_int i586_ctr_comultiplier; extern u_int i586_ctr_freq; @@ -81,7 +81,7 @@ clock_latency(void) - ((high << 8) | low)); } -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * When we update `time', on i586's we also update `i586_ctr_bias' * atomically. `i586_ctr_bias' is the best available approximation to diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h index 9cd03517e3a1..740b7b23765f 100644 --- a/sys/i386/include/cpufunc.h +++ b/sys/i386/include/cpufunc.h @@ -30,7 +30,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: cpufunc.h,v 1.61 1997/02/22 09:34:08 peter Exp $ + * $Id: cpufunc.h,v 1.62 1997/03/22 18:52:57 kato Exp $ */ /* @@ -42,6 +42,10 @@ #include <sys/cdefs.h> #include <sys/types.h> +#include <machine/smp.h> + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" #ifdef __GNUC__ @@ -201,6 +205,16 @@ invd(void) __asm __volatile("invd"); } +#if defined(SMP) && defined(SMP_INVLTLB) + +/* + * When using APIC IPI's, the inlining cost is prohibitive.. + */ +void invlpg __P((u_int addr)); +void invltlb __P((void)); + +#else + static __inline void invlpg(u_int addr) { @@ -218,6 +232,7 @@ invltlb(void) __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) : : "memory"); } +#endif /* SMP && SMP_INVLTLB */ static __inline u_short inw(u_int port) diff --git a/sys/i386/include/ipl.h b/sys/i386/include/ipl.h index 773fa9c700c6..09e72627e1cc 100644 --- a/sys/i386/include/ipl.h +++ b/sys/i386/include/ipl.h @@ -30,13 +30,30 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id$ + * $Id: ipl.h,v 1.4 1997/02/22 09:34:45 peter Exp $ */ #ifndef _ISA_IPL_H_ #define _ISA_IPL_H_ +#include "opt_smp.h" + +#if defined(APIC_IO) + +#include <machine/apic.h> +#if defined(IPI_INTS) +#define NHWI 28 /* number of h/w interrupts */ +#define HWI_MASK 0x0fffffff /* bits for h/w interrupts */ +#else +#define NHWI 24 /* number of h/w interrupts */ +#define HWI_MASK 0x00ffffff /* bits for h/w interrupts */ +#endif /* IPI_INTS */ + +#else + #define NHWI 16 /* number of h/w interrupts */ #define HWI_MASK 0xffff /* bits corresponding to h/w interrupts */ +#endif /* APIC_IO */ + #endif /* _ISA_IPL_H_ */ diff --git a/sys/i386/include/mpapic.h b/sys/i386/include/mpapic.h new file mode 100644 index 000000000000..c3e7c6eea0e4 --- /dev/null +++ b/sys/i386/include/mpapic.h @@ -0,0 +1,338 @@ +/* + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: mpapic.h,v 1.14 1997/04/26 08:11:50 peter Exp $ + */ + +#ifndef _MACHINE_MPAPIC_H_ +#define _MACHINE_MPAPIC_H_ + +#include <i386/isa/icu.h> + +/* number of busses */ +#if !defined(NBUS) +# define NBUS 4 +#endif /* NBUS */ + +/* total number of APIC INTs, including SHARED INTs */ +#if !defined(NINTR) +#define NINTR 24 +#endif /* NINTR */ + +/* size of APIC ID list */ +#define NAPICID 16 + +/* number of IO APICs */ +# if !defined(NAPIC) +# define NAPIC 1 +# endif /* NAPIC */ + +/* use inline xxxIPI functions */ +#define FAST_IPI_NOT +#define APICIPI_BANDAID + + +/* these don't really belong in here... */ +enum busTypes { + CBUS = 1, + CBUSII = 2, + EISA = 3, + ISA = 6, + PCI = 13, + XPRESS = 18, + MAX_BUSTYPE = 18, + UNKNOWN_BUSTYPE = 0xff +}; + + +/* + * the physical/logical APIC ID management macors + */ +#define CPU_TO_ID(CPU) (cpu_num_to_apic_id[CPU]) +#define ID_TO_CPU(ID) (apic_id_to_logical[ID]) +#define IO_TO_ID(IO) (io_num_to_apic_id[IO]) +#define ID_TO_IO(ID) (apic_id_to_logical[ID]) + + +/* + * inline functions to read/write the IO APIC + * NOTES: + * unlike the local APIC, the IO APIC is accessed indirectly thru 2 registers. + * the select register is loaded with an index to the desired 'window' reg. + * the 'window' is accessed as a 32 bit unsigned. + */ + +/* + * read 'reg' from 'apic' + */ +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX +#else +static __inline u_int32_t +io_apic_read(int apic __attribute__ ((unused)), int reg) + +{ + (*io_apic_base) = reg; + return (*(io_apic_base + (IOAPIC_WINDOW / sizeof(u_int)))); +} +#endif /* MULTIPLE_IOAPICS */ + + +/* + * write 'value' to 'reg' of 'apic' + */ +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX +#else +static __inline void +io_apic_write(int apic __attribute__ ((unused)), int reg, u_int32_t value) +{ + (*io_apic_base) = reg; + (*(io_apic_base + (IOAPIC_WINDOW / sizeof(u_int)))) = value; +} +#endif /* MULTIPLE_IOAPICS */ + + +#if defined(READY) +/* + * set the IO APIC mask for INT# 'i' + */ +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX +#else +static __inline void +set_io_apic_mask(int apic, u_int32_t i) +{ + int select; /* the select register is 8 bits */ + u_int32_t low_reg; /* the window register is 32 bits */ + + imen |= (1<<i); /* set mask variable */ + + select = IOAPIC_REDTBL + (i * 2); /* calculate addr */ + low_reg = io_apic_read(select); /* read contents */ + + low_reg |= IOART_INTMASK; /* set mask */ + io_apic_write(select, low_reg); /* new value */ +} +#endif /* MULTIPLE_IOAPICS */ +#endif /* READY */ + + +#if defined(READY) +/* + * clear the IO APIC mask for INT# 'i' + */ +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX +#else +static __inline void +clr_io_apic_mask(int apic, u_int32_t i) +{ + int select; /* the select register is 8 bits */ + u_int32_t low_reg; /* the window register is 32 bits */ + + imen &= ~(1<<i); /* clear mask variable */ + + select = IOAPIC_REDTBL + (i * 2); /* calculate addr */ + low_reg = io_apic_read(select); /* read contents */ + + low_reg &= ~IOART_INTMASK; /* clear mask */ + io_apic_write(select, low_reg); /* new value */ +} +#endif /* MULTIPLE_IOAPICS */ +#endif /* READY */ + + +/* + * read current IRQ0 -IRQ23 masks + */ +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX +#else +static __inline u_int32_t +read_io_apic_mask24(int apic __attribute__ ((unused))) +{ + return (imen & 0x00ffffff); /* return our global copy */ +} +#endif /* MULTIPLE_IOAPICS */ + + +/* + * send an EndOfInterrupt to the local APIC + */ +static __inline void +apic_eoi(void) +{ + apic_base[APIC_EOI] = 0; +} + + +#if defined(FAST_IPI) + +/* + * send APIC IPI 'vector' to 'destType' via 'deliveryMode' + * + * destType is 1 of: APIC_DEST_SELF, APIC_DEST_ALLISELF, APIC_DEST_ALLESELF + * vector is any valid SYSTEM INT vector + * deliveryMode is 1 of: APIC_DELMODE_FIXED, APIC_DELMODE_LOWPRIO + */ +static __inline int +apic_ipi(int destType, int vector, int deliveryMode) +{ + u_long icr_lo; + + /* build IRC_LOW */ + icr_lo = (apic_base[APIC_ICR_LOW] & APIC_RESV2_MASK) | + destType | deliveryMode | vector; + + /* write APIC ICR */ + apic_base[APIC_ICR_LOW] = icr_lo; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /** FIXME: return result */ + return 0; +} + + +/* + * send an IPI INTerrupt containing 'vector' to CPU 'target' + * NOTE: target is a LOGICAL APIC ID + */ +static __inline int +selected_proc_ipi(int target, int vector) +{ + u_long icr_lo; + u_long icr_hi; + + /* write the destination field for the target AP */ + icr_hi = (apic_base[APIC_ICR_HI] & ~APIC_ID_MASK) | + (cpu_num_to_apic_id[target] << 24); + apic_base[APIC_ICR_HI] = icr_hi; + + /* write command */ + icr_lo = (apic_base[APIC_ICR_LOW] & APIC_RESV2_MASK) | + APIC_DEST_DESTFLD | APIC_DELMODE_FIXED | vector; + apic_base[APIC_ICR_LOW] = icr_lo; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + return 0; /** FIXME: return result */ +} + + +/* + * send an IPI INTerrupt containing 'vector' to CPUs in 'targetMap' + * 'targetMap' is a bitfiled of length 14, + * APIC #0 == bit 0, ..., APIC #14 == bit 14 + * NOTE: these are LOGICAL APIC IDs + */ +static __inline int +selected_procs_ipi(int targetMap, int vector) +{ + return selected_apic_ipi(targetMap, vector, APIC_DELMODE_FIXED); +} + +/* + * send an IPI INTerrupt containing 'vector' to all CPUs, including myself + */ +static __inline int +all_procs_ipi(int vector) +{ + u_int32_t icr_lo; + + /* build command */ + icr_lo = (apic_base[APIC_ICR_LOW] & APIC_RESV2_MASK) | + APIC_DEST_ALLISELF | APIC_DELMODE_FIXED | vector; + + /* write command */ + apic_base[APIC_ICR_LOW] = icr_lo; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + return 0; /** FIXME: return result */ +} + + +/* + * send an IPI INTerrupt containing 'vector' to all CPUs EXCEPT myself + */ +static __inline int +all_but_self_ipi(int vector) +{ + u_int32_t icr_lo; + + /* build command */ + icr_lo = (apic_base[APIC_ICR_LOW] & APIC_RESV2_MASK) | + APIC_DEST_ALLESELF | APIC_DELMODE_FIXED | vector; + + /* write command */ + apic_base[APIC_ICR_LOW] = icr_lo; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + return 0; /** FIXME: return result */ +} + +/* + * send an IPI INTerrupt containing 'vector' to myself + */ +static __inline int +self_ipi(int vector) +{ + u_int32_t icr_lo; + + /* build command */ + icr_lo = (apic_base[APIC_ICR_LOW] & APIC_RESV2_MASK) | + APIC_DEST_SELF | APIC_DELMODE_FIXED | vector; + + /* write command */ + apic_base[APIC_ICR_LOW] = icr_lo; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + return 0; /** FIXME: return result */ +} + +# else /* !FAST_IPI */ + +int apic_ipi __P((int, int, int)); +int selected_procs_ipi __P((int, int)); +int all_procs_ipi __P((int)); +int all_but_self_ipi __P((int)); +int self_ipi __P((int)); + +#endif /* FAST_IPI */ + +#endif /* _MACHINE_MPAPIC_H */ diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h new file mode 100644 index 000000000000..07fca34efe56 --- /dev/null +++ b/sys/i386/include/mptable.h @@ -0,0 +1,1646 @@ +/* + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: mp_machdep.c,v 1.47 1997/04/26 08:11:49 peter Exp $ + */ + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" +#if defined(APIC_IO) +#if !defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#else /* APIC_IO */ +#if defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#endif /* APIC_IO */ +#define FIX_MP_TABLE_WORKS_NOT + +#include "opt_serial.h" + +#include <sys/param.h> /* for KERNBASE */ +#include <sys/types.h> +#include <sys/sysproto.h> +#include <sys/time.h> +#include <sys/systm.h> + +#include <vm/vm.h> /* for KERNBASE */ +#include <vm/vm_param.h> /* for KERNBASE */ +#include <vm/pmap.h> /* for KERNBASE */ +#include <machine/pmap.h> /* for KERNBASE */ + +#include <machine/smp.h> +#include <machine/apic.h> +#include <machine/mpapic.h> +#include <machine/cpufunc.h> +#include <machine/segments.h> +#include <machine/smptests.h> /** TEST_UPPERPRIO, TEST_DEFAULT_CONFIG */ + +#include <i386/i386/cons.h> /* cngetc() */ + +#if defined(IPI_INTS) +#include <i386/isa/isa_device.h> +#include "vector.h" +#endif /* IPI_INTS */ + +#if defined(SMP_INVLTLB) +#include <i386/isa/icu.h> +#endif /* SMP_INVLTLB */ + +#define WARMBOOT_TARGET 0 +#define WARMBOOT_OFF (KERNBASE + 0x0467) +#define WARMBOOT_SEG (KERNBASE + 0x0469) + +#define BIOS_BASE (0xf0000) +#define BIOS_SIZE (0x10000) +#define BIOS_COUNT (BIOS_SIZE/4) + +#define CMOS_REG (0x70) +#define CMOS_DATA (0x71) +#define BIOS_RESET (0x0f) +#define BIOS_WARM (0x0a) + +/* + * this code MUST be enabled here and in mpboot.s. + * it follows the very early stages of AP boot by placing values in CMOS ram. + * it NORMALLY will never be needed and thus the primitive method for enabling. + * +#define CHECK_POINTS + */ + +#if defined(CHECK_POINTS) +#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) +#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) + +#define CHECK_INIT(D); \ + CHECK_WRITE(0x34, (D)); \ + CHECK_WRITE(0x35, (D)); \ + CHECK_WRITE(0x36, (D)); \ + CHECK_WRITE(0x37, (D)); \ + CHECK_WRITE(0x38, (D)); \ + CHECK_WRITE(0x39, (D)); + +#define CHECK_PRINT(S); \ + printf("%s: %d, %d, %d, %d, %d, %d\n", \ + (S), \ + CHECK_READ(0x34), \ + CHECK_READ(0x35), \ + CHECK_READ(0x36), \ + CHECK_READ(0x37), \ + CHECK_READ(0x38), \ + CHECK_READ(0x39)); + +#else /* CHECK_POINTS */ + +#define CHECK_INIT(D) +#define CHECK_PRINT(S) + +#endif /* CHECK_POINTS */ + + +/** FIXME: what system files declare these??? */ +extern struct region_descriptor r_gdt, r_idt; + +/* global data */ +struct proc *SMPcurproc[NCPU]; +struct pcb *SMPcurpcb[NCPU]; +struct timeval SMPruntime[NCPU]; + +int mp_ncpus; /* # of CPUs, including BSP */ +int mp_naps; /* # of Applications processors */ +int mp_nbusses; /* # of busses */ +int mp_napics; /* # of IO APICs */ +int mpenabled; +int boot_cpu_id; /* designated BSP */ +vm_offset_t cpu_apic_address; +vm_offset_t io_apic_address[NAPIC]; + +u_int32_t cpu_apic_versions[NCPU]; +u_int32_t io_apic_versions[NAPIC]; + +/* + * APIC ID logical/physical mapping structures + */ +int cpu_num_to_apic_id[NCPU]; +int io_num_to_apic_id[NAPIC]; +int apic_id_to_logical[NAPICID]; + +/* + * look for MP compliant motherboard. + */ + +static u_int boot_address; +static u_int base_memory; + +static int picmode; /* 0: virtual wire mode, 1: PIC mode */ +static u_int mpfps; +static int search_for_sig(u_int32_t target, int count); +static int mp_probe(u_int base_top); +static void mp_enable(u_int boot_addr); + +#if defined(IPI_INTS) +static void ipi_intr0(void); +static void ipi_intr1(void); +static void ipi_intr2(void); +static void ipi_intr3(void); +static int +ipi_ihandler_attach(int irq, inthand2_t * func, + unsigned *maskptr, int unit); +#endif /* IPI_INTS */ + + +/* + * calculate usable address in base memory for AP trampoline code + */ +u_int +mp_bootaddress(u_int basemem) +{ + base_memory = basemem * 1024; /* convert to bytes */ + + boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ + if ((base_memory - boot_address) < bootMP_size) + boot_address -= 4096; /* not enough, lower by 4k */ + + return boot_address; +} + + +/* + * startup the SMP processors + */ +void +mp_start(void) +{ + /* look for MP capable motherboard */ + if (mp_probe(base_memory)) + mp_enable(boot_address); + else { + printf("MP FPS NOT FOUND, suggest use of 'mptable' program\n"); + panic("can't continue!\n"); + } + + /* finish pmap initialization - turn off V==P mapping at zero */ + pmap_bootstrap2(); +} + + +/* + * print various information about the SMP system hardware and setup + */ +void +mp_announce(void) +{ + int x; + + printf("FreeBSD/SMP: Multiprocessor motherboard\n"); + printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); + printf(", version: 0x%08x\n", cpu_apic_versions[0]); + for (x = 1; x <= mp_naps; ++x) { + printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); + printf(", version: 0x%08x\n", cpu_apic_versions[x]); + } + +#if defined(APIC_IO) + for (x = 0; x < mp_napics; ++x) { + printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); + printf(", version: 0x%08x\n", io_apic_versions[x]); + } +#else + printf(" Warning: APIC I/O disabled\n"); +#endif /* APIC_IO */ +} + + +/* + * AP cpu's call this to sync up protected mode. + */ +void +init_secondary(void) +{ + int gsel_tss, slot; + + r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; + r_gdt.rd_base = (int) gdt; + lgdt(&r_gdt); /* does magic intra-segment return */ + lidt(&r_idt); + lldt(_default_ldt); + + slot = NGDT + cpunumber(); + gsel_tss = GSEL(slot, SEL_KPL); + gdt[slot].sd.sd_type = SDT_SYS386TSS; + ltr(gsel_tss); + + load_cr0(0x8005003b); /* XXX! */ +} + + +#if defined(APIC_IO) +void +configure_local_apic(void) +{ + u_char byte; + u_int32_t temp; + + if (picmode) { + outb(0x22, 0x70); /* select IMCR */ + byte = inb(0x23); /* current contents */ + byte |= 0x01; /* mask external INTR */ + outb(0x23, byte); /* disconnect 8259s/NMI */ + } + /* mask the LVT1 */ + temp = apic_base[APIC_LVT1]; + temp |= APIC_LVT_M; + apic_base[APIC_LVT1] = temp; +} +#endif /* APIC_IO */ + + +/******************************************************************* + * local functions and data + */ + +static int +mp_probe(u_int base_top) +{ + int x; + u_long segment; + u_int32_t target; + + /* see if EBDA exists */ + if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { + /* search first 1K of EBDA */ + target = (u_int32_t) (segment << 4); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } else { + /*last 1K of base memory, effective 'top of base' is passed in*/ + target = (u_int32_t) (base_top - 0x400); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } + + /* search the BIOS */ + target = (u_int32_t) BIOS_BASE; + if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) + goto found; + + /* nothing found */ + mpfps = mpenabled = 0; + return 0; + +found: /* please forgive the 'goto'! */ + /* flag fact that we are running multiple processors */ + mpfps = x; + mpenabled = 1; + return 1; +} + + +/* + * start the SMP system + */ +static int parse_mp_table(void); +static void default_mp_table(int type); +static int start_all_aps(u_int boot_addr); + +#if defined(XFAST_IPI32) +#include <machine/md_var.h> +#include <i386/isa/isa_device.h> +extern void Xfastipi32(u_int, u_int, u_int, u_int); +#endif /* XFAST_IPI32 */ + +static void +mp_enable(u_int boot_addr) +{ + int x; +#if defined(APIC_IO) + int apic; + u_int ux; +#if defined(TEST_UPPERPRIO) + u_char select; /* the select register is 8 bits */ + u_int32_t flags; /* the window register is 32 bits */ +#endif /* TEST_UPPERPRIO */ +#endif /* APIC_IO */ + + /* examine the MP table for needed info */ + x = parse_mp_table(); + + /* create pages for (address common) cpu APIC and each IO APIC */ + pmap_bootstrap_apics(); + + /* can't process default configs till the CPU APIC is pmapped */ + if (x) + default_mp_table(x); + +#if defined(APIC_IO) + /* fill the LOGICAL io_apic_versions table */ + for (apic = 0; apic < mp_napics; ++apic) { + ux = io_apic_read(apic, IOAPIC_VER); + io_apic_versions[apic] = ux; + } + + /* + */ + for (apic = 0; apic < mp_napics; ++apic) + if (io_apic_setup(apic) < 0) + panic("IO APIC setup failure\n"); + +#if defined(IPI_INTS) + /* setup IPI INTerrupt mechanism */ + ipi_ihandler_attach( /* irq */ 24, + /* XXX */ (inthand2_t *) ipi_intr0, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 25, + /* XXX */ (inthand2_t *) ipi_intr1, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 26, + /* XXX */ (inthand2_t *) ipi_intr2, NULL, /* unit */ 0); +#if !defined(SMP_INVLTLB) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); +#else +#if defined(XFAST_IPI32) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); + setidt(ICU_OFFSET + 32, + Xfastipi32, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#else + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_invltlb, NULL, /* unit */ 0); +#endif /* XFAST_IPI32 */ +#endif + +#endif /* IPI_INTS */ + +#if defined(TEST_UPPERPRIO) + +#if 1 + printf("special IRQ10\n"); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~0xff; /** clear vector */ + flags |= 64; + io_apic_write(0, select, flags); +#else + printf("special IRQ10\n"); + cngetc(); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~IOART_DELMOD; /* FIXED mode */ + io_apic_write(0, select, flags); + io_apic_write(0, select + 1, boot_cpu_id << 24); +#endif /** 0/1 */ + +#endif /* TEST_UPPERPRIO */ + +#endif /* APIC_IO */ + + /* start each Application Processor */ + start_all_aps(boot_addr); +} + + +/* + * look for the MP spec signature + */ + +/* string defined by the Intel MP Spec as identifying the MP table */ +#define MP_SIG 0x5f504d5f /* _MP_ */ +#define NEXT(X) ((X) += 4) +static int +search_for_sig(u_int32_t target, int count) +{ + int x; + u_int32_t *addr = (u_int32_t *) (KERNBASE + target); + + for (x = 0; x < count; NEXT(x)) + if (addr[x] == MP_SIG) + /* make array index a byte index */ + return (target + (x * sizeof(u_int32_t))); + + return -1; +} + + +#define PROCENTRY_FLAG_EN 0x01 +#define PROCENTRY_FLAG_BP 0x02 +#define IOAPICENTRY_FLAG_EN 0x01 + +/* MP Floating Pointer Structure */ +typedef struct MPFPS { + char signature[4]; + void *pap; + u_char length; + u_char spec_rev; + u_char checksum; + u_char mpfb1; + u_char mpfb2; + u_char mpfb3; + u_char mpfb4; + u_char mpfb5; +} *mpfps_t; +/* MP Configuration Table Header */ +typedef struct MPCTH { + char signature[4]; + u_short base_table_length; + u_char spec_rev; + u_char checksum; + u_char oem_id[8]; + u_char product_id[12]; + void *oem_table_pointer; + u_short oem_table_size; + u_short entry_count; + void *apic_address; + u_short extended_table_length; + u_char extended_table_checksum; + u_char reserved; +} *mpcth_t; + + +typedef struct PROCENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char cpu_flags; + u_long cpu_signature; + u_long feature_flags; + u_long reserved1; + u_long reserved2; +} *proc_entry_ptr; + +typedef struct BUSENTRY { + u_char type; + u_char bus_id; + char bus_type[6]; +} *bus_entry_ptr; + +typedef struct IOAPICENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char apic_flags; + void *apic_address; +} *io_apic_entry_ptr; + +typedef struct INTENTRY { + u_char type; + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} *int_entry_ptr; +/* descriptions of MP basetable entries */ +typedef struct BASETABLE_ENTRY { + u_char type; + u_char length; + char name[16]; +} basetable_entry; + +static basetable_entry basetable_entry_types[] = +{ + {0, 20, "Processor"}, + {1, 8, "Bus"}, + {2, 8, "I/O APIC"}, + {3, 8, "I/O INT"}, + {4, 8, "Local INT"} +}; + +typedef struct BUSDATA { + u_char bus_id; + enum busTypes bus_type; +} bus_datum; + +typedef struct INTDATA { + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} io_int, local_int; + +typedef struct BUSTYPENAME { + u_char type; + char name[7]; +} bus_type_name; + +static bus_type_name bus_type_table[] = +{ + {CBUS, "CBUS"}, + {CBUSII, "CBUSII"}, + {EISA, "EISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {ISA, "ISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {PCI, "PCI"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {XPRESS, "XPRESS"}, + {UNKNOWN_BUSTYPE, "---"} +}; +/* from MP spec v1.4, table 5-1 */ +static int default_data[7][5] = +{ +/* nbus, id0, type0, id1, type1 */ + {1, 0, ISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {0, 255, 255, 255, 255},/* MCA not supported */ + {2, 0, ISA, 1, PCI}, + {2, 0, EISA, 1, PCI}, + {0, 255, 255, 255, 255} /* MCA not supported */ +}; + + +/* the bus data */ +bus_datum bus_data[NBUS]; + +/* the IO INT data, one entry per possible APIC INTerrupt */ +io_int io_apic_ints[NINTR]; + +static int nintrs; + +#if defined(FIX_MP_TABLE_WORKS) +static void fix_mp_table __P((void)); +#endif /* FIX_MP_TABLE_WORKS */ + +static void processor_entry __P((proc_entry_ptr entry, int *cpu)); +static void io_apic_entry __P((io_apic_entry_ptr entry, int *apic)); +static void bus_entry __P((bus_entry_ptr entry, int *bus)); +static void int_entry __P((int_entry_ptr entry, int *intr)); +static int lookup_bus_type __P((char *name)); + + +/* + * parse an Intel MP specification table + */ +static int +parse_mp_table(void) +{ + int x; + mpfps_t fps; + mpcth_t cth; + int totalSize; + void *position; + int count; + int type; + int apic, bus, cpu, intr; + + /* clear physical APIC ID to logical CPU/IO table */ + for (x = 0; x < NAPICID; ++x) + ID_TO_IO(x) = -1; + + /* clear logical CPU to APIC ID table */ + for (x = 0; x < NCPU; ++x) + CPU_TO_ID(x) = -1; + + /* clear logical IO to APIC ID table */ + for (x = 0; x < NAPIC; ++x) + IO_TO_ID(x) = -1; + + /* clear IO APIC address table */ + for (x = 0; x < NAPIC; ++x) + io_apic_address[x] = ~0; + + /* clear bus data table */ + for (x = 0; x < NBUS; ++x) + bus_data[x].bus_id = 0xff; + + /* clear IO APIC INT table */ + for (x = 0; x < NINTR; ++x) + io_apic_ints[x].int_type = 0xff; + nintrs = 0; + + /* count the BSP */ + mp_ncpus = 1; + + /* setup the cpu/apic mapping arrays */ + boot_cpu_id = -1; + + /* local pointer */ + fps = (mpfps_t) mpfps; + + /* record whether PIC or virtual-wire mode */ + picmode = (fps->mpfb2 & 0x80) ? 1 : 0; + + /* check for use of 'default' configuration */ +#if defined(TEST_DEFAULT_CONFIG) + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return TEST_DEFAULT_CONFIG; +#else + if (fps->mpfb1 != 0) { + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return fps->mpfb1; + } +#endif /* TEST_DEFAULT_CONFIG */ + + if ((cth = fps->pap) == 0) + panic("MP Configuration Table Header MISSING!\n"); + + cpu_apic_address = (vm_offset_t) cth->apic_address; + + totalSize = cth->base_table_length - sizeof(struct MPCTH); + position = (u_char *) cth + sizeof(struct MPCTH); + count = cth->entry_count; + + apic = 0; /* logical apic# start @ 0 */ + bus = 0; /* logical bus# start @ 0 */ + cpu = 1; /* logical cpu# start @ 0, BUT reserve 0 for */ + /* BSP */ + intr = 0; /* unknown */ + + /* walk the table, recording info of interest */ + while (count--) { + switch (type = *(u_char *) position) { + case 0: + processor_entry(position, &cpu); + break; + case 1: + bus_entry(position, &bus); + break; + case 2: + io_apic_entry(position, &apic); + break; + case 3: + int_entry(position, &intr); + break; + case 4: + /* int_entry(position); */ + break; + default: + panic("mpfps Base Table HOSED!\n"); + /* NOTREACHED */ + } + + totalSize -= basetable_entry_types[type].length; + (u_char *) position += basetable_entry_types[type].length; + } + + if (boot_cpu_id == -1) + panic("NO BSP found!\n"); + + /* record # of APs found */ + mp_naps = (cpu - 1); + + /* record # of busses found */ + mp_nbusses = bus; + + /* record # of IO APICs found */ + mp_napics = apic; + + /* record # of IO APICs found */ + nintrs = intr; + +#if defined(FIX_MP_TABLE_WORKS) + /* post scan cleanup */ + fix_mp_table(); +#endif /* FIX_MP_TABLE_WORKS */ + + /* report fact that its NOT a default configuration */ + return 0; +} + + +/* + * parse an Intel MP specification table + */ +#if defined(FIX_MP_TABLE_WORKS) +static void +fix_mp_table(void) +{ + int x; + int y; + int num_pci_bus; + bus_datum bus_record; + + /* + * Fix mis-numbering of the PCI bus and its INT entries if the BIOS + * did it wrong. The MP spec says that when more than 1 PCI bus + * exists the BIOS must begin with bus entries for the PCI bus and use + * actual PCI bus numbering. This implies that when only 1 PCI bus + * exists the BIOS can choose to ignore this ordering, and indeed many + * MP motherboards do ignore it. This causes a problem when the PCI + * sub-system makes requests of the MP sub-system based on PCI bus + * numbers. So here we look for the situation and renumber the + * busses and associated INTs in an effort to "make it right". + */ + + /* count the number of PCI busses */ + for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + ++num_pci_bus; + } + + /* check the 1 PCI bus case for sanity */ + if (num_pci_bus == 1) { + + /* if its in the first slot all is well */ + if (bus_data[0].bus_type == PCI) + return; + + /* mis-numbered, swap with whichever bus uses slot 0 */ + + /* locate the entry holding the PCI bus */ + for (x = 1; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + break; + } + + /* swap the bus entry records */ + bus_record = bus_data[0]; + bus_data[0] = bus_data[x]; + bus_data[x] = bus_record; + + /* swap each relavant INTerrupt entry */ + for (y = 0; y < nintrs; ++y) { + if (io_apic_ints[y].src_bus_id == x) + io_apic_ints[y].src_bus_id = 0; + else + if (io_apic_ints[y].src_bus_id == 0) + io_apic_ints[y].src_bus_id = x; + } + } + /* sanity check if more than 1 PCI bus */ + else + if (num_pci_bus > 1) { + for (x = 0; x < num_pci_bus; ++x) { + if (bus_data[x].bus_type != PCI) { + printf("bad PCI bus numbering\n"); + panic("\n"); + } + } + } +} +#endif /* FIX_MP_TABLE_WORKS */ + + +static void +processor_entry(proc_entry_ptr entry, int *cpu) +{ + int x = *cpu; + + /* check for usability */ + if (!(entry->cpu_flags & PROCENTRY_FLAG_EN)) + return; + + /* check for BSP flag */ + if (entry->cpu_flags & PROCENTRY_FLAG_BP) { + /* always give boot CPU the logical value of 0 */ + x = 0; + boot_cpu_id = entry->apic_id; + } else { + /* add another AP to list, if less than max number of CPUs */ + if (x == NCPU) { + printf("Warning: only using %d of the available CPUs!\n", x); + return; + } + ++(*cpu); + } + + CPU_TO_ID(x) = entry->apic_id; + ID_TO_CPU(entry->apic_id) = x; +} + + +static void +bus_entry(bus_entry_ptr entry, int *bus) +{ + int x, y; + char name[8]; + char c; + + if ((x = (*bus)++) == NBUS) + panic("too many busses, increase 'NBUS'\n"); + + /* encode the name into an index */ + for (y = 0; y < 6; ++y) { + if ((c = entry->bus_type[y]) == ' ') + break; + name[y] = c; + } + name[y] = '\0'; + + if ((y = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) + panic("unknown bus type: '%s'\n", name); + + bus_data[x].bus_id = entry->bus_id; + bus_data[x].bus_type = y; +} + + +static void +io_apic_entry(io_apic_entry_ptr entry, int *apic) +{ + int x; + + if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) + return; + + if ((x = (*apic)++) == NAPIC) + panic("too many APICs, increase 'NAPIC'\n"); + + IO_TO_ID(x) = entry->apic_id; + ID_TO_IO(entry->apic_id) = x; + + io_apic_address[x] = (vm_offset_t) entry->apic_address; +} + + +static int +lookup_bus_type(char *name) +{ + int x; + + for (x = 0; x < MAX_BUSTYPE; ++x) + if (strcmp(bus_type_table[x].name, name) == 0) + return bus_type_table[x].type; + + return UNKNOWN_BUSTYPE; +} + + +static void +int_entry(int_entry_ptr entry, int *intr) +{ + int x; + + if ((x = (*intr)++) == NINTR) + panic("too many INTs, increase 'NINTR'\n"); + + io_apic_ints[x].int_type = entry->int_type; + io_apic_ints[x].int_flags = entry->int_flags; + io_apic_ints[x].src_bus_id = entry->src_bus_id; + io_apic_ints[x].src_bus_irq = entry->src_bus_irq; + io_apic_ints[x].dst_apic_id = entry->dst_apic_id; + io_apic_ints[x].dst_apic_int = entry->dst_apic_int; +} + + +static int +apic_int_is_bus_type(int intr, int bus_type) +{ + int bus; + + for (bus = 0; bus < mp_nbusses; ++bus) + if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) + && ((int) bus_data[bus].bus_type == bus_type)) + return 1; + + return 0; +} + + +/* + * determine which APIC pin an ISA INT is attached to. + */ +#define INTTYPE(I) (io_apic_ints[(I)].int_type) +#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) + +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_isa_apic_irq(int isaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (isaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == isaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, ISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin an EISA INT is attached to. + */ +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_eisa_apic_irq(int eisaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (eisaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == eisaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, EISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin a PCI INT is attached to. + */ +#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) +#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) +#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) +int +get_pci_apic_irq(int pciBus, int pciDevice, int pciInt) +{ + int intr; + + --pciInt; /* zero based */ + + for (intr = 0; intr < nintrs; ++intr) /* search each record */ + if ((INTTYPE(intr) == 0) +#if defined(FIX_MP_TABLE_WORKS) + && (SRCBUSID(intr) == pciBus) +#endif /* FIX_MP_TABLE_WORKS */ + && (SRCBUSDEVICE(intr) == pciDevice) + && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, PCI)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSLINE +#undef SRCBUSDEVICE +#undef SRCBUSID + +#undef INTPIN +#undef INTTYPE + + +int +undirect_pci_irq(int rirq) +{ +#if defined(READY) + printf("Freeing irq %d for ISA cards.\n", rirq); + /** FIXME: tickle the MB redirector chip */ + return ???; +#else + printf("Freeing (NOT implimented) irq %d for ISA cards.\n", rirq); + return 0; +#endif /* READY */ +} + + +/* + * given a bus ID, return: + * the bus type if found + * -1 if NOT found + */ +int +apic_bus_type(int id) +{ + int x; + + for (x = 0; x < mp_nbusses; ++x) + if (bus_data[x].bus_id == id) + return bus_data[x].bus_type; + + return -1; +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus ID if found + * -1 if NOT found + */ +int +apic_src_bus_id(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_id); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus IRQ if found + * -1 if NOT found + */ +int +apic_src_bus_irq(int apic, int pin) +{ + int x; + + for (x = 0; x < nintrs; x++) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_irq); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated INTerrupt type if found + * -1 if NOT found + */ +int +apic_int_type(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_type); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated trigger mode if found + * -1 if NOT found + */ +int +apic_trigger(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return ((io_apic_ints[x].int_flags >> 2) & 0x03); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated 'active' level if found + * -1 if NOT found + */ +int +apic_polarity(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_flags & 0x03); + + return -1; /* NOT found */ +} + + +/* + * set data according to MP defaults + * FIXME: probably not complete yet... + */ +static void +default_mp_table(int type) +{ + int ap_cpu_id; +#if defined(APIC_IO) + u_int32_t ux; + int io_apic_id; + int pin; +#endif /* APIC_IO */ + +#if 0 + printf(" MP default config type: %d\n", type); + switch (type) { + case 1: + printf(" bus: ISA, APIC: 82489DX\n"); + break; + case 2: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 3: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 4: + printf(" bus: MCA, APIC: 82489DX\n"); + break; + case 5: + printf(" bus: ISA+PCI, APIC: Integrated\n"); + break; + case 6: + printf(" bus: EISA+PCI, APIC: Integrated\n"); + break; + case 7: + printf(" bus: MCA+PCI, APIC: Integrated\n"); + break; + default: + printf(" future type\n"); + break; + /* NOTREACHED */ + } +#endif /* 0 */ + + boot_cpu_id = (apic_base[APIC_ID] & APIC_ID_MASK) >> 24; + ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; + + /* BSP */ + CPU_TO_ID(0) = boot_cpu_id; + ID_TO_CPU(boot_cpu_id) = 0; + + /* one and only AP */ + CPU_TO_ID(1) = ap_cpu_id; + ID_TO_CPU(ap_cpu_id) = 1; + mp_naps = 1; + + /* one and only IO APIC */ +#if defined(APIC_IO) + io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; + + /* + * sanity check, refer to MP spec section 3.6.6, last paragraph + * necessary as some hardware isn't properly setting up the IO APIC + */ +#if defined(REALLY_ANAL_IOAPICID_VALUE) + if (io_apic_id != 2) { +#else + if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { +#endif /* REALLY_ANAL_IOAPICID_VALUE */ + ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ + ux &= ~APIC_ID_MASK; /* clear the ID field */ + ux |= 0x02000000; /* set it to '2' */ + io_apic_write(0, IOAPIC_ID, ux); /* write new value */ + ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ + if ((ux & APIC_ID_MASK) != 0x02000000) + panic("Problem: can't control IO APIC ID, reg: 0x%08x\n", ux); + io_apic_id = 2; + } + IO_TO_ID(0) = io_apic_id; + ID_TO_IO(io_apic_id) = 0; + mp_napics = 1; +#else + mp_napics = 0; +#endif /* APIC_IO */ + + /* fill out bus entries */ + switch (type) { + case 1: + case 2: + case 3: + case 5: + case 6: + mp_nbusses = default_data[type - 1][0]; + bus_data[0].bus_id = default_data[type - 1][1]; + bus_data[0].bus_type = default_data[type - 1][2]; + bus_data[1].bus_id = default_data[type - 1][3]; + bus_data[1].bus_type = default_data[type - 1][4]; + break; + + /* case 4: case 7: MCA NOT supported */ + default: /* illegal/reserved */ + panic("BAD default MP config: %d\n", type); + } + +#if defined(APIC_IO) + /* general cases from MP v1.4, table 5-2 */ + for (pin = 0; pin < 16; ++pin) { + io_apic_ints[pin].int_type = 0; + io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ + io_apic_ints[pin].src_bus_id = 0; + io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ + io_apic_ints[pin].dst_apic_id = io_apic_id; + io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ + } + + /* special cases from MP v1.4, table 5-2 */ + if (type == 2) { + io_apic_ints[2].int_type = 0xff; /* N/C */ + io_apic_ints[13].int_type = 0xff; /* N/C */ +#if !defined(APIC_MIXED_MODE) + /** FIXME: ??? */ + panic("sorry, can't support type 2 default yet\n"); +#endif /* APIC_MIXED_MODE */ + } else + io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ + + if (type == 7) + io_apic_ints[0].int_type = 0xff; /* N/C */ + else + io_apic_ints[0].int_type = 3; /* vectored 8259 */ + + nintrs = 16; +#endif /* APIC_IO */ +} + + +static void install_ap_tramp(u_int boot_addr); +static int start_ap(int logicalCpu, u_int boot_addr); + +/* + * start each AP in our list + */ +static int +start_all_aps(u_int boot_addr) +{ + int x; + u_char mpbiosreason; + u_long mpbioswarmvec; + + /** + * NOTE: this needs further thought: + * where does it get released? + * should it be set to empy? + * + * get the initial mp_lock with a count of 1 for the BSP + */ + mp_lock = (apic_base[APIC_ID] & APIC_ID_MASK) + 1; + + /* initialize BSP's local APIC */ + apic_initialize(1); + + /* install the AP 1st level boot code */ + install_ap_tramp(boot_addr); + + /* save the current value of the warm-start vector */ + mpbioswarmvec = *((u_long *) WARMBOOT_OFF); + outb(CMOS_REG, BIOS_RESET); + mpbiosreason = inb(CMOS_DATA); + + /* start each AP */ + for (x = 1; x <= mp_naps; ++x) { + + /* setup a vector to our boot code */ + *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; + *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ + + /* attempt to start the Application Processor */ + CHECK_INIT(99); /* setup checkpoints */ + if (!start_ap(x, boot_addr)) { + printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); + CHECK_PRINT("trace"); /* show checkpoints */ + /* + * better panic as the AP may be running loose + * somewhere + */ + printf("panic y/n? [n] "); + if (cngetc() != 'n') + panic("bye-bye\n"); + } + CHECK_PRINT("trace"); /* show checkpoints */ + + /* record its version info */ + cpu_apic_versions[x] = cpu_apic_versions[0]; + } + + /* fill in our (BSP) APIC version */ + cpu_apic_versions[0] = apic_base[APIC_VER]; + + /* restore the warmstart vector */ + *(u_long *) WARMBOOT_OFF = mpbioswarmvec; + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, mpbiosreason); + + /* number of APs actually started */ + return mp_ncpus - 1; +} + + +/* + * load the 1st level AP boot code into base memory. + */ + +/* targets for relocation */ +extern void bigJump(void); +extern void bootCodeSeg(void); +extern void bootDataSeg(void); +extern void MPentry(void); +extern u_int MP_GDT; +extern u_int mp_gdtbase; + +static void +install_ap_tramp(u_int boot_addr) +{ + int x; + int size = *(int *) ((u_long) & bootMP_size); + u_char *src = (u_char *) ((u_long) bootMP); + u_char *dst = (u_char *) boot_addr + KERNBASE; + u_int boot_base = (u_int) bootMP; + u_int8_t *dst8; + u_int16_t *dst16; + u_int32_t *dst32; + + for (x = 0; x < size; ++x) + *dst++ = *src++; + + /* + * modify addresses in code we just moved to basemem. unfortunately we + * need fairly detailed info about mpboot.s for this to work. changes + * to mpboot.s might require changes here. + */ + + /* boot code is located in KERNEL space */ + dst = (u_char *) boot_addr + KERNBASE; + + /* modify the lgdt arg */ + dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); + *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); + + /* modify the ljmp target for MPentry() */ + dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); + *dst32 = ((u_int) MPentry - KERNBASE); + + /* modify the target for boot code segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; + + /* modify the target for boot data segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; +} + + +/* + * this function starts the AP (application processor) identified + * by the APIC ID 'physicalCpu'. It does quite a "song and dance" + * to accomplish this. This is necessary because of the nuances + * of the different hardware we might encounter. It ain't pretty, + * but it seems to work. + */ +static int +start_ap(int logical_cpu, u_int boot_addr) +{ + int physical_cpu; + int vector; + int cpus; + u_long icr_lo, icr_hi; + + /* get the PHYSICAL APIC ID# */ + physical_cpu = CPU_TO_ID(logical_cpu); + + /* calculate the vector */ + vector = (boot_addr >> 12) & 0xff; + + /* used as a watchpoint to signal AP startup */ + cpus = mp_ncpus; + + /* + * first we do an INIT/RESET IPI this INIT IPI might be run, reseting + * and running the target CPU. OR this INIT IPI might be latched (P5 + * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be + * ignored. + */ + + /* setup the address for the target AP */ + icr_hi = apic_base[APIC_ICR_HI] & ~APIC_ID_MASK; + icr_hi |= (physical_cpu << 24); + apic_base[APIC_ICR_HI] = icr_hi; + + /* do an INIT IPI: assert RESET */ + icr_lo = apic_base[APIC_ICR_LOW] & 0xfff00000; + apic_base[APIC_ICR_LOW] = icr_lo | 0x0000c500; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* do an INIT IPI: deassert RESET */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00008500; + + /* wait for pending status end */ + u_sleep(10000); /* wait ~10mS */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* + * next we do a STARTUP IPI: the previous INIT IPI might still be + * latched, (P5 bug) this 1st STARTUP would then terminate + * immediately, and the previously started INIT IPI would continue. OR + * the previous INIT IPI has already run. and this STARTUP IPI will + * run. OR the previous INIT IPI was ignored. and this STARTUP IPI + * will run. + */ + + /* do a STARTUP IPI */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* + * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF + * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR + * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is + * recognized after hardware RESET or INIT IPI. + */ + + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* wait for it to start */ + set_apic_timer(5000000);/* == 5 seconds */ + while (read_apic_timer()) + if (mp_ncpus > cpus) + return 1; /* return SUCCESS */ + + return 0; /* return FAILURE */ +} + + +#if defined(IPI_INTS) + +static void +ipi_intr0(void) +{ + printf("IPI 0\n"); +} + + +static void +ipi_intr1(void) +{ + printf("IPI 1\n"); +} + + +static void +ipi_intr2(void) +{ + printf("IPI 2\n"); +} + + +static void +ipi_intr3(void) +{ + printf("IPI 3\n"); +} + +/*----------------------------------------------------------------------- +** +** Register an interupt handler for an IPI. +** (Stolen from the PCI<->ISA glue code) +** +**----------------------------------------------------------------------- +*/ + +static int +ipi_ihandler_attach(int irq, inthand2_t * func, unsigned *maskptr, int unit) +{ + char buf[16]; + char *cp; + int free_id, id, result; + + sprintf(buf, "ipi irq%d", irq); + for (cp = intrnames, free_id = 0, id = 0; id < NR_DEVICES; id++) { + if (strcmp(cp, buf) == 0) + break; + if (free_id <= 0 && strcmp(cp, "ipi irqnn") == 0) + free_id = id; + while (*cp++ != '\0'); + } + if (id == NR_DEVICES) { + id = free_id; + if (id == 0) { + /* + * All ipi irq counters are in use, perhaps because + * config is old so there aren't any. Abuse the clk0 + * counter. + */ + printf( + "ipi_ihandler_attach: counting ipi irq%d's as clk0 irqs\n", + irq); + } + } + result = register_intr( + irq, /* isa irq */ + id, /* device id */ + 0, /* flags? */ + func, /* handler */ + maskptr, /* mask pointer */ + unit); /* handler arg */ + + if (result) { + printf("WARNING: ipi_ihandler_attach: result=%d\n", result); + return (result); + }; + + return (0); +} +#endif /* IPI_INTS */ + + +#ifdef SMP_INVLTLB +/* + * Flush the TLB on all other CPU's + * + * XXX: Needs to handshake and wait for completion before proceding. + */ + +void +smp_invltlb() +{ + if (smp_active) { + if (invldebug & 2) +#if defined(XFAST_IPI32) + all_but_self_ipi(ICU_OFFSET + 32); +#else + all_but_self_ipi(ICU_OFFSET + 27); +#endif /* XFAST_IPI32 */ + } +} + +void +invlpg(u_int addr) +{ + __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); + smp_invltlb(); +} + +void +invltlb(void) +{ + u_long temp; + /* + * This should be implemented as load_cr3(rcr3()) when load_cr3() is + * inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); + smp_invltlb(); +} +/* + * Handles recieving an "IRQ 27", the invalidate tlb IPI.. + */ +void +ipi_invltlb(void) +{ + u_long temp; + + if (invldebug & 4) { + /* + * This should be implemented as load_cr3(rcr3()) when + * load_cr3() is inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) + :: "memory"); + + } +} +#endif /* SMP_INVLTLB */ diff --git a/sys/i386/include/pcb.h b/sys/i386/include/pcb.h index ecfd8b2c315d..73ee223404cc 100644 --- a/sys/i386/include/pcb.h +++ b/sys/i386/include/pcb.h @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)pcb.h 5.10 (Berkeley) 5/12/91 - * $Id: pcb.h,v 1.18 1997/02/22 09:34:56 peter Exp $ + * $Id: pcb.h,v 1.19 1997/04/07 06:45:18 peter Exp $ */ #ifndef _I386_PCB_H_ @@ -60,6 +60,8 @@ struct pcb { u_char pcb_flags; #define FP_SOFTFP 0x01 /* process using software fltng pnt emulator */ caddr_t pcb_onfault; /* copyin/out fault recovery */ + u_long pcb_mpnest; + u_long __pcb_spare[7]; /* adjust to avoid core dump size changes */ #if 0 /* some day we may switch between procs that have their own i386tss */ struct i386tss pcb_tss; u_char pcb_iomap[NPORT/sizeof(u_char)]; /* i/o port bitmap */ @@ -74,7 +76,15 @@ struct md_coredump { }; #ifdef KERNEL + +#include "opt_smp.h" +#include <machine/smp.h> +#ifdef SMP +#define curpcb (SMPcurpcb[cpunumber()]) +#else /* !SMP */ extern struct pcb *curpcb; /* our current running pcb */ +#endif /* SMP */ + void savectx __P((struct pcb*)); #endif diff --git a/sys/i386/include/pmap.h b/sys/i386/include/pmap.h index 6b27cd28ac34..f0d4c017178b 100644 --- a/sys/i386/include/pmap.h +++ b/sys/i386/include/pmap.h @@ -42,7 +42,7 @@ * * from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90 * from: @(#)pmap.h 7.4 (Berkeley) 5/12/91 - * $Id: pmap.h,v 1.48 1997/02/22 09:34:58 peter Exp $ + * $Id: pmap.h,v 1.49 1997/04/07 09:30:20 peter Exp $ */ #ifndef _MACHINE_PMAP_H_ @@ -91,7 +91,11 @@ #define NKPT 9 /* actual number of kernel page tables */ #endif #ifndef NKPDE +#if defined(SMP) && defined(SMP_PRIVPAGES) +#define NKPDE 62 /* addressable number of page tables/pde's */ +#else /* SMP && SMP_PRIVPAGES */ #define NKPDE 63 /* addressable number of page tables/pde's */ +#endif /* SMP && SMP_PRIVPAGES */ #endif /* @@ -99,9 +103,16 @@ * * XXX This works for now, but I am not real happy with it, I'll fix it * right after I fix locore.s and the magic 28K hole + * + * SMP_PRIVPAGES: The per-cpu address space is 0xff80000 -> 0xffbfffff */ #define APTDPTDI (NPDEPG-1) /* alt ptd entry that points to APTD */ +#if defined(SMP) && defined(SMP_PRIVPAGES) +#define MPPTDI (APTDPTDI-1) /* per cpu ptd entry */ +#define KPTDI (MPPTDI-NKPDE) /* start of kernel virtual pde's */ +#else /* SMP && SMP_PRIVPAGES */ #define KPTDI (APTDPTDI-NKPDE)/* start of kernel virtual pde's */ +#endif /* SMP && SMP_PRIVPAGES */ #define PTDPTDI (KPTDI-1) /* ptd entry that points to ptd! */ #define UMAXPTDI (PTDPTDI-1) /* ptd entry for user space end */ #define UMAXPTEOFF (NPTEPG-UPAGES_HOLE) /* pte entry for user space end */ diff --git a/sys/i386/include/segments.h b/sys/i386/include/segments.h index 27525889053d..e1e6b4684c91 100644 --- a/sys/i386/include/segments.h +++ b/sys/i386/include/segments.h @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)segments.h 7.1 (Berkeley) 5/9/91 - * $Id$ + * $Id: segments.h,v 1.13 1997/02/22 09:35:09 peter Exp $ */ #ifndef _MACHINE_SEGMENTS_H_ @@ -234,7 +234,7 @@ struct region_descriptor { #ifdef KERNEL extern int currentldt; extern int _default_ldt; -extern union descriptor gdt[NGDT]; +extern union descriptor gdt[]; extern struct soft_segment_descriptor gdt_segs[]; extern struct gate_descriptor idt[NIDT]; extern union descriptor ldt[NLDT]; diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h new file mode 100644 index 000000000000..905ca3529327 --- /dev/null +++ b/sys/i386/include/smp.h @@ -0,0 +1,135 @@ +/* + * ---------------------------------------------------------------------------- + * "THE BEER-WARE LICENSE" (Revision 42): + * <phk@FreeBSD.org> wrote this file. As long as you retain this notice you + * can do whatever you want with this stuff. If we meet some day, and you think + * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp + * ---------------------------------------------------------------------------- + * + * $Id: smp.h,v 1.29 1997/04/26 08:11:50 peter Exp $ + * + */ + +#ifndef _MACHINE_SMP_H_ +#define _MACHINE_SMP_H_ + +#ifdef KERNEL + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" /* aiee! (for cpufunc.h!) */ + +#if defined(SMP) && !defined(NCPU) +# define NCPU 2 +#endif /* SMP && NCPU */ + +#if defined(SMP) || defined(APIC_IO) + +/* global data in mpboot.s */ +extern int bootMP_size; + +/* functions in mpboot.s */ +void bootMP __P((void)); + +/* global data in mplock.s */ +extern u_int mp_lock; + +/* functions in mplock.s */ +void get_mplock __P((void)); +void rel_mplock __P((void)); +void expect_mplock __P((void)); + +/* global data in mp_machdep.c */ +extern struct proc* SMPcurproc[NCPU]; +extern struct pcb* SMPcurpcb[NCPU]; +extern struct timeval SMPruntime[NCPU]; +extern int mp_ncpus; +extern int mp_naps; +extern int mp_nbusses; +extern int mp_napics; +extern int mp_picmode; +extern int mpenabled; +extern int boot_cpu_id; +extern vm_offset_t cpu_apic_address; +extern vm_offset_t io_apic_address[]; +extern u_int32_t cpu_apic_versions[]; +extern u_int32_t io_apic_versions[]; +extern int cpu_num_to_apic_id[]; +extern int io_num_to_apic_id[]; +extern int apic_id_to_logical[]; + +/* functions in mp_machdep.c */ +u_int mp_bootaddress __P((u_int)); +void mp_start __P((void)); +void mp_announce __P((void)); +int get_isa_apic_irq __P((int)); +int get_eisa_apic_irq __P((int)); +int get_pci_apic_irq __P((int, int, int)); +int undirect_pci_irq __P((int)); +int apic_bus_type __P((int)); +int apic_src_bus_id __P((int, int)); +int apic_src_bus_irq __P((int, int)); +int apic_int_type __P((int, int)); +int apic_trigger __P((int, int)); +int apic_polarity __P((int, int)); +void configure_local_apic __P((void)); +void init_secondary __P((void)); +#ifdef SMP_INVLTLB +void ipi_invltlb __P((void)); +void smp_invltlb __P((void)); +#endif + +/* global data in mpapic.c */ +extern volatile u_int* apic_base; + +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX +#else +extern volatile u_int* io_apic_base; +#endif /* MULTIPLE_IOAPICS */ + +/* functions in mpapic.c */ +void apic_initialize __P((int)); +int selected_apic_ipi __P((u_int, int, int)); +int io_apic_setup __P((int)); +int ext_int_setup __P((int, int)); +void write_io_apic_mask24 __P((int, u_int32_t)); + +#if defined(READY) +void clr_io_apic_mask24 __P((int, u_int32_t)); +void set_io_apic_mask24 __P((int, u_int32_t)); +#endif /* READY */ + +void set_apic_timer __P((int)); +int read_apic_timer __P((void)); +void u_sleep __P((int)); + +/* global data in init_smp.c */ +extern int smp_active; +extern int invldebug; + +/* in pmap.c FIXME: belongs in pmap.h??? */ +void pmap_bootstrap_apics __P((void)); +void pmap_bootstrap2 __P((void)); + +#if 0 +/* chicken and egg problem... */ +static __inline unsigned +cpunumber(void) +{ + return (unsigned)ID_TO_CPU((apic_base[APIC_ID] & APIC_ID_MASK) >> 24); +} +#else +/* + * we 'borrow' this info from apic.h + * this will go away soon... + */ +static __inline unsigned +cpunumber(void) +{ + return (unsigned)(apic_id_to_logical[(apic_base[8] & 0x0f000000) >> 24]); +} +#endif /* 0 */ + +#endif /* SMP || APIC_IO */ +#endif /* KERNEL */ +#endif /* _MACHINE_SMP_H_ */ diff --git a/sys/i386/include/smpasm.h b/sys/i386/include/smpasm.h new file mode 100644 index 000000000000..6f15307c955c --- /dev/null +++ b/sys/i386/include/smpasm.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 1996, by Peter Wemm + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: smpasm.h,v 1.8 1997/04/25 03:11:40 fsmp Exp $ + */ +#ifndef __MACHINE_SMPASM_H__ +#define __MACHINE_SMPASM_H__ 1 + + +#if defined(SMP) || defined(APIC_IO) + +#include <machine/apic.h> + +/* Macro to retrieve the current CPU id from hardware */ +#define GETCPUID(reg) \ + movl _apic_base, reg; \ + movl APIC_ID(reg), reg; \ + andl $APIC_ID_MASK, reg; \ + shrl $24, reg; \ + movl _apic_id_to_logical(,reg,4), reg + +#define SETCURPROC(val, reg) GETCPUID(reg); movl val, _SMPcurproc(,reg,4) +#define GETCURPROC(reg) GETCPUID(reg); movl _SMPcurproc(,reg,4), reg +#define SETCURPCB(val, reg) GETCPUID(reg); movl val, _SMPcurpcb(,reg,4) +#define GETCURPCB(reg) GETCPUID(reg); movl _SMPcurpcb(,reg,4), reg +#define SETNPXPROC(val, reg) GETCPUID(reg); movl val, _SMPnpxproc(,reg,4) +#define GETNPXPROC(reg) GETCPUID(reg); movl _SMPnpxproc(,reg,4), reg + +#else /* !SMP && !APIC_IO */ + +#define SETCURPROC(val, reg) movl val, _curproc +#define GETCURPROC(reg) movl _curproc, reg +#define SETCURPCB(val, reg) movl val, _curpcb +#define GETCURPCB(reg) movl _curpcb, reg +#define SETNPXPROC(val, reg) movl val, _npxproc +#define GETNPXPROC(reg) movl _npxproc, reg + +#endif /* SMP || APIC_IO */ + +#endif /* __MACHINE_SMPASM_H__ */ diff --git a/sys/i386/include/smptests.h b/sys/i386/include/smptests.h new file mode 100644 index 000000000000..99d79a9d2bdb --- /dev/null +++ b/sys/i386/include/smptests.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: smptests.h,v 1.13 1997/04/26 07:24:21 fsmp Exp $ + */ + +#ifndef _MACHINE_SMPTESTS_H_ +#define _MACHINE_SMPTESTS_H_ + + +/* + * various 'tests in progress' + */ + + +/* + * faster invltlb IPI, asm only + */ +#define XFAST_IPI32 + + +/* + * use 'lowest priority' for sending IRQs to CPUs + * + * i386/i386/mplock.s, i386/i386/mpapic.c, kern/init_main.c + * + */ +#define TEST_LOPRIO + + +/* + * test the InterProcessor Interrupt mechanism. + * + * i386/i386/mpapic.c + */ +#define TEST_IPI + + +/* + * count INT hits by CPU + * + * i386/isa/vector.s + * + */ +#define TEST_CPUHITS + + +/* + * deal with broken smp_idleloop() + */ +#define IGNORE_IDLEPROCS + + +/** + * hack to "fake-out" kernel into thinking it is running on a 'default config' + * + * value == default type +#define TEST_DEFAULT_CONFIG 6 + */ + + +/** + * hacks attempting to assign IRQs to upper IDT vectors, doesn't work yet. + * + * i386/isa/icu.s, i386/isa/isa.c, i386/i386/mpapic.c + * +#define TEST_UPPERPRIO + */ + + +#endif /* _MACHINE_SMPTESTS_H_ */ diff --git a/sys/i386/include/spl.h b/sys/i386/include/spl.h index 6f305acc2d56..e9be52a064f2 100644 --- a/sys/i386/include/spl.h +++ b/sys/i386/include/spl.h @@ -30,12 +30,13 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id$ + * $Id: spl.h,v 1.17 1997/02/22 09:35:16 peter Exp $ */ #ifndef _MACHINE_IPL_H_ #define _MACHINE_IPL_H_ +#include <opt_smp.h> #include <machine/ipl.h> /* XXX "machine" means cpu for i386 */ /* @@ -43,8 +44,14 @@ * determines which swi will be dispatched next; a higher priority swi * may be dispatched when a nested h/w interrupt handler returns. */ +#if defined(APIC_IO) +#define SWI_TTY 28 +#define SWI_NET 29 +#else #define SWI_TTY (NHWI + 0) #define SWI_NET (NHWI + 1) +#endif /* APIC_IO */ + #define SWI_CLOCK 30 #define SWI_AST 31 @@ -90,14 +97,28 @@ extern unsigned stat_imask; /* interrupts masked with splstatclock() */ extern unsigned tty_imask; /* group of interrupts masked with spltty() */ /* + * ipending has to be volatile so that it is read every time it is accessed + * in splx() and spl0(), but we don't want it to be read nonatomically when + * it is changed. Pretending that ipending is a plain int happens to give + * suitable atomic code for "ipending |= constant;". + * * The volatile bitmap variables must be set atomically. This normally * involves using a machine-dependent bit-set or `or' instruction. */ -#define setdelayed() setbits(&ipending, loadandclear(&idelayed)) -#define setsoftast() setbits(&ipending, SWI_AST_PENDING) -#define setsoftclock() setbits(&ipending, SWI_CLOCK_PENDING) -#define setsoftnet() setbits(&ipending, SWI_NET_PENDING) -#define setsofttty() setbits(&ipending, SWI_TTY_PENDING) +static __inline void +SETBITS(unsigned foo) +{ +#ifdef SMP + __asm __volatile("lock ; orl %0, _ipending" : : "a" (foo) ); +#else + __asm __volatile("orl %0, _ipending" : : "a" (foo) ); +#endif /* SMP */ +} +#define setdelayed() SETBITS(loadandclear(&idelayed)) +#define setsoftast() SETBITS(SWI_AST_PENDING) +#define setsoftclock() SETBITS(SWI_CLOCK_PENDING) +#define setsoftnet() SETBITS(SWI_NET_PENDING) +#define setsofttty() SETBITS(SWI_TTY_PENDING) #define schedsofttty() setbits(&idelayed, SWI_TTY_PENDING) #define schedsoftnet() setbits(&idelayed, SWI_NET_PENDING) diff --git a/sys/i386/isa/clock.c b/sys/i386/isa/clock.c index b9425536d67a..d46f982806bd 100644 --- a/sys/i386/isa/clock.c +++ b/sys/i386/isa/clock.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)clock.c 7.2 (Berkeley) 5/12/91 - * $Id: clock.c,v 1.79 1997/03/05 08:08:48 bde Exp $ + * $Id: clock.c,v 1.80 1997/04/06 13:25:48 mckay Exp $ */ /* @@ -49,6 +49,7 @@ */ #include "opt_clock.h" +#include "opt_smp.h" #include "opt_cpu.h" #include <sys/param.h> @@ -100,7 +101,7 @@ int adjkerntz; /* local offset from GMT in seconds */ int disable_rtc_set; /* disable resettodr() if != 0 */ u_int idelayed; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) u_int i586_ctr_bias; u_int i586_ctr_comultiplier; u_int i586_ctr_freq; @@ -143,7 +144,7 @@ static u_char timer0_state; static u_char timer2_state; static void (*timer_func) __P((struct clockframe *frame)) = hardclock; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) static void set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq); #endif static void set_timer_freq(u_int freq, int intr_freq); @@ -553,7 +554,7 @@ calibrate_clocks(void) goto fail; tot_count = 0; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) if (cpu_class == CPUCLASS_586 || cpu_class == CPUCLASS_686) wrmsr(0x10, 0LL); /* XXX 0x10 is the MSR for the TSC */ #endif @@ -586,7 +587,7 @@ calibrate_clocks(void) goto fail; } -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * Read the cpu cycle counter. The timing considerations are * similar to those for the i8254 clock. @@ -667,14 +668,14 @@ startrtclock() printf( "%d Hz differs from default of %d Hz by more than 1%%\n", freq, timer_freq); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) i586_ctr_freq = 0; #endif } set_timer_freq(timer_freq, hz); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) #ifndef CLK_USE_I586_CALIBRATION if (i586_ctr_freq != 0) { if (bootverbose) @@ -821,6 +822,20 @@ resettodr() writertc(RTC_STATUSB, rtc_statusb); } +#if defined(APIC_IO) + +/* from icu.s: */ +extern u_int hwisrs[]; +extern void vec8254 __P((void)); +extern void vecRTC __P((void)); +extern u_int ivectors[]; +extern u_int Xintr8254; +extern u_int XintrRTC; +extern u_int mask8254; +extern u_int maskRTC; + +#endif /* APIC_IO */ + /* * Start both clocks running. */ @@ -828,6 +843,9 @@ void cpu_initclocks() { int diag; +#if defined(APIC_IO) + int x; +#endif /* APIC_IO */ if (statclock_disable) { /* @@ -845,11 +863,47 @@ cpu_initclocks() } /* Finish initializing 8253 timer 0. */ +#if defined(APIC_IO) + /* 8254 is traditionally on ISA IRQ0 */ + if ((x = get_isa_apic_irq(0)) < 0) { + /* + * bummer, this mb doesn't have the 8254 on ISA irq0, + * perhaps it's on the EISA bus... + */ + if ((x = get_eisa_apic_irq(0)) < 0) { + /* double bummer, attempt to redirect thru the 8259 */ + if (bootverbose) + printf("APIC missing 8254 connection\n"); + + /* allow 8254 timer to INTerrupt 8259 */ +#if !defined(IO_ICU1) +#define IO_ICU1 0x20 +#endif + x = inb(IO_ICU1 + 1); /* current mask in 8259 */ + x &= ~1; /* clear 8254 timer mask */ + outb(IO_ICU1 + 1, x); /* write new mask */ + + /* program IO APIC for type 3 INT on INT0 */ + if (ext_int_setup(0, 0) < 0) + panic("8254 redirect impossible!"); + x = 0; /* 8259 is on 0 */ + } + } + + hwisrs[x] = (u_int)vec8254; + Xintr8254 = (u_int)ivectors[x]; /* XXX might need Xfastintr# */ + mask8254 = (1 << x); + register_intr(/* irq */ x, /* XXX id */ 0, /* flags */ 0, + /* XXX */ (inthand2_t *)clkintr, &clk_imask, + /* unit */ 0); + INTREN(mask8254); +#else register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0, /* XXX */ (inthand2_t *)clkintr, &clk_imask, /* unit */ 0); INTREN(IRQ0); -#if defined(I586_CPU) || defined(I686_CPU) +#endif /* APIC_IO */ +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * Finish setting up anti-jitter measures. */ @@ -867,11 +921,36 @@ cpu_initclocks() diag = rtcin(RTC_DIAG); if (diag != 0) printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS); +#if defined(APIC_IO) + /* RTC is traditionally on ISA IRQ8 */ + if ((x = get_isa_apic_irq(8)) < 0) { + if ((x = get_eisa_apic_irq(8)) < 0) { + panic("APIC missing RTC connection"); + } + } + + hwisrs[x] = (u_int)vecRTC; + XintrRTC = (u_int)ivectors[x]; /* XXX might need Xfastintr# */ + maskRTC = (1 << x); + register_intr(/* irq */ x, /* XXX id */ 1, /* flags */ 0, + /* XXX */ (inthand2_t *)rtcintr, &stat_imask, + /* unit */ 0); + INTREN(maskRTC); +#else register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0, /* XXX */ (inthand2_t *)rtcintr, &stat_imask, /* unit */ 0); INTREN(IRQ8); +#endif /* APIC_IO */ writertc(RTC_STATUSB, rtc_statusb); + +#if defined(APIC_IO) + printf("Enabled INTs: "); + for (x = 0; x < 24; ++x) + if ((imen & (1 << x)) == 0) + printf("%d, ", x); + printf("imen: 0x%08x\n", imen); +#endif /* APIC_IO */ } void @@ -900,7 +979,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS if (timer0_state != 0) return (EBUSY); /* too much trouble to handle */ set_timer_freq(freq, hz); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) set_i586_ctr_freq(i586_ctr_freq, timer_freq); #endif } @@ -910,7 +989,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS SYSCTL_PROC(_machdep, OID_AUTO, i8254_freq, CTLTYPE_INT | CTLFLAG_RW, 0, sizeof(u_int), sysctl_machdep_i8254_freq, "I", ""); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) static void set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq) { @@ -949,4 +1028,4 @@ sysctl_machdep_i586_freq SYSCTL_HANDLER_ARGS SYSCTL_PROC(_machdep, OID_AUTO, i586_freq, CTLTYPE_INT | CTLFLAG_RW, 0, sizeof(u_int), sysctl_machdep_i586_freq, "I", ""); -#endif /* defined(I586_CPU) || defined(I686_CPU) */ +#endif /* (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) */ diff --git a/sys/i386/isa/icu.h b/sys/i386/isa/icu.h index ba721727f340..d0f148289dc3 100644 --- a/sys/i386/isa/icu.h +++ b/sys/i386/isa/icu.h @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)icu.h 5.6 (Berkeley) 5/9/91 - * $Id$ + * $Id: icu.h,v 1.10 1997/02/22 09:36:13 peter Exp $ */ /* @@ -47,15 +47,62 @@ #ifndef LOCORE +#include "opt_smp.h" /* so we dont have to change EVERY file including icu.h */ + /* * Interrupt "level" mechanism variables, masks, and macros */ extern unsigned imen; /* interrupt mask enable */ +#if defined(APIC_IO) + +# if !defined(_MACHINE_SMP_H_) +/** XXX what a hack, its this or include <machine/smp.h>! */ +void write_io_apic_mask24 __P((int, u_int32_t)); /* i386/i386/mpapic.c */ +# endif /* _MACHINE_SMP_H_ */ + +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX: cannot assume apic #0 in the following functions. +#endif /* MULTIPLE_IOAPICS */ + +static __inline u_int32_t +INTRGET( void ) +{ + return (imen & 0x00ffffff); /* return our global copy */ +} + +static __inline void +INTRSET( unsigned s ) +{ + write_io_apic_mask24( 0, s ); + imen = s; +} + +static __inline void +INTREN( unsigned s ) +{ + write_io_apic_mask24( 0, imen & ~s ); + imen &= ~s; +} + +static __inline void +INTRDIS( unsigned s ) +{ + write_io_apic_mask24( 0, imen | s ); + imen |= s; +} + + +#define INTRMASK(msk,s) (msk |= (s)) +#define INTRUNMASK(msk,s) (msk &= ~(s)) + +#else /* APIC_IO */ + #define INTREN(s) (imen &= ~(s), SET_ICUS()) #define INTRDIS(s) (imen |= (s), SET_ICUS()) #define INTRMASK(msk,s) (msk |= (s)) #define INTRUNMASK(msk,s) (msk &= ~(s)) + #if 0 #ifdef PC98 #define SET_ICUS() (outb(IO_ICU1 + 2, imen), outb(IU_ICU2 + 2, imen >> 8)) @@ -74,6 +121,8 @@ extern unsigned imen; /* interrupt mask enable */ #endif #endif +#endif /* APIC_IO */ + #endif /* LOCORE */ /* @@ -108,6 +157,22 @@ extern unsigned imen; /* interrupt mask enable */ * Interrupt Control offset into Interrupt descriptor table (IDT) */ #define ICU_OFFSET 32 /* 0-31 are processor exceptions */ + +#if defined(APIC_IO) + +#include <machine/apic.h> +#if defined(IPI_INTS) +/* 32-47: ISA IRQ0-IRQ15, 48-55: IO APIC IRQ16-IRQ23, 56-59: LOCAL APIC IPI */ +#define ICU_LEN 28 +#else +/* 32-47: ISA IRQ0-IRQ15, 48-55: IO APIC IRQ16-IRQ23 */ +#define ICU_LEN 24 +#endif /* IPI_INTS */ + +#else + #define ICU_LEN 16 /* 32-47 are ISA interrupts */ +#endif /* APIC_IO */ + #endif /* !_I386_ISA_ICU_H_ */ diff --git a/sys/i386/isa/icu.s b/sys/i386/isa/icu.s index a326740c8dcf..e8134b1cf080 100644 --- a/sys/i386/isa/icu.s +++ b/sys/i386/isa/icu.s @@ -36,9 +36,16 @@ * * @(#)icu.s 7.2 (Berkeley) 5/21/91 * - * $Id$ + * $Id: icu.s,v 1.29 1997/02/22 09:36:14 peter Exp $ */ +#include "opt_smp.h" + +#if defined(SMP) +#include <machine/smptests.h> /** TEST_UPPERPRIO */ +#endif /* SMP */ + + /* * AT/386 * Vector interrupt control section @@ -75,9 +82,39 @@ _netisrs: .long dummynetisr, dummynetisr, dummynetisr, dummynetisr .long dummynetisr, dummynetisr, dummynetisr, dummynetisr .long dummynetisr, dummynetisr, dummynetisr, dummynetisr + +#if defined(APIC_IO) + /* this allows us to change the 8254 APIC pin# assignment */ + .globl _Xintr8254 +_Xintr8254: + .long _Xintr7 + + /* this allows us to change the RTC clock APIC pin# assignment */ + .globl _XintrRTC +_XintrRTC: + .long _Xintr7 + + /* used by this file, microtime.s and clock.c */ + .globl _mask8254 +_mask8254: + .long 0 + + /* used by this file and clock.c */ + .globl _maskRTC +_maskRTC: + .long 0 + + + /* this allows us to change ISA IRQ# vs APIC pin# assignments */ + .globl _hwisrs +_hwisrs: +#endif /* APIC_IO */ vec: .long vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7 .long vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15 +#if defined(APIC_IO) + .long vec16, vec17, vec18, vec19, vec20, vec21, vec22, vec23 +#endif /* APIC_IO */ .text @@ -108,6 +145,9 @@ doreti_exit: movl %eax,_cpl decb _intr_nesting_level MEXITCOUNT +#ifdef SMP + call _rel_mplock +#endif /* SMP */ .globl doreti_popl_es doreti_popl_es: popl %es @@ -145,6 +185,9 @@ doreti_unpend: */ sti bsfl %ecx,%ecx /* slow, but not worth optimizing */ +#ifdef SMP + lock +#endif btrl %ecx,_ipending jnc doreti_next /* some intr cleared memory copy */ movl ihandlers(,%ecx,4),%edx @@ -197,6 +240,9 @@ swi_ast_phantom: * using by using cli, but they are unavoidable for lcall entries. */ cli +#ifdef SMP + lock +#endif orl $SWI_AST_PENDING,_ipending subl %eax,%eax jmp doreti_exit /* SWI_AST is highest so we must be done */ @@ -235,6 +281,9 @@ splz_next: ALIGN_TEXT splz_unpend: bsfl %ecx,%ecx +#ifdef SMP + lock +#endif btrl %ecx,_ipending jnc splz_next movl ihandlers(,%ecx,4),%edx @@ -270,6 +319,23 @@ splz_swi: * XXX frame bogusness stops us from just jumping to the C entry point. */ ALIGN_TEXT +#if defined(APIC_IO) + /* generic vector function for 8254 clock */ + .globl _vec8254 +_vec8254: + popl %eax /* return address */ + pushfl +#define KCSEL 8 + pushl $KCSEL + pushl %eax + cli + movl _mask8254,%eax /* lazy masking */ + notl %eax + andl %eax,iactive + MEXITCOUNT + movl _Xintr8254, %eax + jmp %eax /* XXX might need _Xfastintr# */ +#else /* APIC_IO */ vec0: popl %eax /* return address */ pushfl @@ -279,9 +345,26 @@ vec0: cli MEXITCOUNT jmp _Xintr0 /* XXX might need _Xfastintr0 */ +#endif /* APIC_IO */ #ifndef PC98 ALIGN_TEXT +#if defined(APIC_IO) + /* generic vector function for RTC clock */ + .globl _vecRTC +_vecRTC: + popl %eax + pushfl + pushl $KCSEL + pushl %eax + cli + movl _maskRTC,%eax /* lazy masking */ + notl %eax + andl %eax,iactive + MEXITCOUNT + movl _XintrRTC, %eax + jmp %eax /* XXX might need _Xfastintr# */ +#else vec8: popl %eax pushfl @@ -290,13 +373,28 @@ vec8: cli MEXITCOUNT jmp _Xintr8 /* XXX might need _Xfastintr8 */ -#endif +#endif /* APIC_IO */ +#endif /* PC98 */ +# if defined(APIC_IO) +#define BUILD_VEC(irq_num) \ + ALIGN_TEXT ; \ +__CONCAT(vec,irq_num): ; \ + popl %eax ; \ + pushfl ; \ + pushl $KCSEL ; \ + pushl %eax ; \ + cli ; \ + andl $~IRQ_BIT(irq_num),iactive ; /* lazy masking */ \ + MEXITCOUNT ; \ + jmp __CONCAT(_Xintr,irq_num) +# else #define BUILD_VEC(irq_num) \ ALIGN_TEXT ; \ __CONCAT(vec,irq_num): ; \ int $ICU_OFFSET + (irq_num) ; \ ret +# endif /* APIC_IO */ BUILD_VEC(1) BUILD_VEC(2) @@ -309,12 +407,30 @@ __CONCAT(vec,irq_num): ; \ BUILD_VEC(8) #endif BUILD_VEC(9) +#if defined(TEST_UPPERPRIO) +vec10: int $64 + ret +#else BUILD_VEC(10) +#endif /* TEST_UPPERPRIO */ BUILD_VEC(11) BUILD_VEC(12) BUILD_VEC(13) BUILD_VEC(14) BUILD_VEC(15) +#if defined(APIC_IO) + BUILD_VEC(0) /* NOT specific in IO APIC hardware */ + BUILD_VEC(8) /* NOT specific in IO APIC hardware */ + + BUILD_VEC(16) /* 8 additional INTs in IO APIC */ + BUILD_VEC(17) + BUILD_VEC(18) + BUILD_VEC(19) + BUILD_VEC(20) + BUILD_VEC(21) + BUILD_VEC(22) + BUILD_VEC(23) +#endif /* APIC_IO */ ALIGN_TEXT swi_net: diff --git a/sys/i386/isa/if_ze.c b/sys/i386/isa/if_ze.c index f34471d65115..ac988a9daa39 100644 --- a/sys/i386/isa/if_ze.c +++ b/sys/i386/isa/if_ze.c @@ -47,7 +47,7 @@ */ /* - * $Id: if_ze.c,v 1.40 1997/02/22 09:36:37 peter Exp $ + * $Id: if_ze.c,v 1.41 1997/03/24 11:32:55 bde Exp $ */ /* XXX - Don't mix different PCCARD support code */ @@ -64,6 +64,7 @@ #include "ze.h" #if NZE > 0 #include "bpfilter.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -116,6 +117,11 @@ #include <machine/apm_bios.h> #endif /* NAPM > 0 */ +#if defined(APIC_IO) +#include <machine/smp.h> +#include <machine/mpapic.h> +#endif /* APIC_IO */ + /***************************************************************************** * Driver for Ethernet Adapter * @@ -727,7 +733,11 @@ ze_watchdog(ifp) #if 1 struct ze_softc *sc = (struct ze_softc *)ifp; u_char isr, imr; +#if defined(APIC_IO) + u_int imask; +#else u_short imask; +#endif /* APIC_IO */ if(!(ifp->if_flags & IFF_UP)) return; @@ -745,7 +755,11 @@ ze_watchdog(ifp) /* read interrupt mask register */ imr = inb (sc->nic_addr + ED_P2_IMR) & 0xff; +#if defined(APIC_IO) + imask = INTRGET(); +#else imask = inb(IO_ICU2) << 8 | inb(IO_ICU1); +#endif /* APIC_IO */ log (LOG_ERR, "ze%d: device timeout, isr=%02x, imr=%02x, imask=%04x\n", ifp->if_unit, isr, imr, imask); diff --git a/sys/i386/isa/isa.c b/sys/i386/isa/isa.c index b4fa75b6665a..8b809e257c8a 100644 --- a/sys/i386/isa/isa.c +++ b/sys/i386/isa/isa.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)isa.c 7.2 (Berkeley) 5/13/91 - * $Id: isa.c,v 1.79 1997/03/25 03:29:40 ache Exp $ + * $Id: isa.c,v 1.80 1997/03/28 01:02:17 ache Exp $ */ /* @@ -48,6 +48,7 @@ */ #include "opt_auto_eoi.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -56,6 +57,10 @@ #include <sys/malloc.h> #include <machine/md_var.h> #include <machine/segments.h> +#if defined(APIC_IO) +#include <machine/smp.h> +#include <machine/apic.h> +#endif /* APIC_IO */ #include <vm/vm.h> #include <vm/vm_param.h> #include <vm/pmap.h> @@ -65,6 +70,14 @@ #include <i386/isa/ic/i8237.h> #include "vector.h" +#ifdef APIC_IO +/* + * This is to accommodate "mixed-mode" programming for + * motherboards that don't connect the 8254 to the IO APIC. + */ +#define AUTO_EOI_1 +#endif + /* ** Register definitions for DMA controller 1 (channels 0..3): */ @@ -96,6 +109,16 @@ static inthand_t *fastintr[ICU_LEN] = { &IDTVEC(fastintr10), &IDTVEC(fastintr11), &IDTVEC(fastintr12), &IDTVEC(fastintr13), &IDTVEC(fastintr14), &IDTVEC(fastintr15) +#if defined(APIC_IO) + , &IDTVEC(fastintr16), &IDTVEC(fastintr17), + &IDTVEC(fastintr18), &IDTVEC(fastintr19), + &IDTVEC(fastintr20), &IDTVEC(fastintr21), + &IDTVEC(fastintr22), &IDTVEC(fastintr23) +#if defined(IPI_INTS) +/* XXX probably NOT needed, we register_intr(slowintr[I]) */ + , &IDTVEC(ipi24), &IDTVEC(ipi25), &IDTVEC(ipi26), &IDTVEC(ipi27) +#endif /* IPI_INTS */ +#endif /* APIC_IO */ }; static inthand_t *slowintr[ICU_LEN] = { @@ -103,6 +126,13 @@ static inthand_t *slowintr[ICU_LEN] = { &IDTVEC(intr4), &IDTVEC(intr5), &IDTVEC(intr6), &IDTVEC(intr7), &IDTVEC(intr8), &IDTVEC(intr9), &IDTVEC(intr10), &IDTVEC(intr11), &IDTVEC(intr12), &IDTVEC(intr13), &IDTVEC(intr14), &IDTVEC(intr15) +#if defined(APIC_IO) + , &IDTVEC(intr16), &IDTVEC(intr17), &IDTVEC(intr18), &IDTVEC(intr19), + &IDTVEC(intr20), &IDTVEC(intr21), &IDTVEC(intr22), &IDTVEC(intr23) +#if defined(IPI_INTS) + , &IDTVEC(ipi24), &IDTVEC(ipi25), &IDTVEC(ipi26), &IDTVEC(ipi27) +#endif /* IPI_INTS */ +#endif /* APIC_IO */ }; static void config_isadev __P((struct isa_device *isdp, u_int *mp)); @@ -920,18 +950,52 @@ struct isa_device *find_isadev(table, driverp, unit) /* * Return nonzero if a (masked) irq is pending for a given device. */ +#if defined(APIC_IO) + int isa_irq_pending(dvp) struct isa_device *dvp; { + /* read APIC IRR containing the 16 ISA INTerrupts */ +#if defined(TEST_UPPERPRIO) + if ((u_int32_t)dvp->id_irq == APIC_IRQ10) + return (int)(apic_base[APIC_IRR2] & 1); + else +#endif /** TEST_UPPERPRIO */ + return ((apic_base[APIC_IRR1] & 0x00ffffff) + & (u_int32_t)dvp->id_irq) ? 1 : 0; +} + +/* + * an 8259 specific routine, + * for use by boot probes in certain device drivers. + */ +int +icu_irq_pending(dvp) + struct isa_device *dvp; +{ unsigned id_irq; + id_irq = dvp->id_irq; + if (id_irq & 0xff) + return (inb(IO_ICU1) & id_irq); + return (inb(IO_ICU2) & (id_irq >> 8)); +} + +#else /* APIC_IO */ +int +isa_irq_pending(dvp) + struct isa_device *dvp; +{ + unsigned id_irq; id_irq = dvp->id_irq; if (id_irq & 0xff) return (inb(IO_ICU1) & id_irq); return (inb(IO_ICU2) & (id_irq >> 8)); } +#endif /* APIC_IO */ + int update_intr_masks(void) { @@ -939,7 +1003,11 @@ update_intr_masks(void) u_int mask,*maskptr; for (intr=0; intr < ICU_LEN; intr ++) { - if (intr==2) continue; +#if defined(APIC_IO) + /* no 8259 SLAVE to ignore */ +#else + if (intr==2) continue; /* ignore 8259 SLAVE output */ +#endif /* APIC_IO */ maskptr = intr_mptr[intr]; if (!maskptr) continue; *maskptr |= 1 << intr; @@ -971,7 +1039,11 @@ register_intr(intr, device_id, flags, handler, maskptr, unit) int id; u_int mask = (maskptr ? *maskptr : 0); +#if defined(APIC_IO) + if ((u_int)intr >= ICU_LEN /* no 8259 SLAVE to ignore */ +#else if ((u_int)intr >= ICU_LEN || intr == 2 +#endif /* APIC_IO */ || (u_int)device_id >= NR_DEVICES) return (EINVAL); if (intr_handler[intr] != isa_strayintr) @@ -983,9 +1055,24 @@ register_intr(intr, device_id, flags, handler, maskptr, unit) intr_mptr[intr] = maskptr; intr_mask[intr] = mask | (1 << intr); intr_unit[intr] = unit; +#if defined(TEST_UPPERPRIO) + if (intr == 10) { + printf("--- setting IRQ10 to IDT64\n"); + setidt(64, + flags & RI_FAST ? fastintr[intr] : slowintr[intr], + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + } + else { + printf("setting IRQ%02d to IDT%02d\n", intr, ICU_OFFSET+intr); + setidt(ICU_OFFSET + intr, + flags & RI_FAST ? fastintr[intr] : slowintr[intr], + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + } +#else setidt(ICU_OFFSET + intr, flags & RI_FAST ? fastintr[intr] : slowintr[intr], SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /** TEST_UPPERPRIO */ write_eflags(ef); for (cp = intrnames, id = 0; id <= device_id; id++) while (*cp++ != '\0') @@ -995,9 +1082,12 @@ register_intr(intr, device_id, flags, handler, maskptr, unit) if (intr < 10) { cp[-3] = intr + '0'; cp[-2] = ' '; - } else { + } else if (intr < 20) { cp[-3] = '1'; cp[-2] = intr - 10 + '0'; + } else { + cp[-3] = '2'; + cp[-2] = intr - 20 + '0'; } return (0); } diff --git a/sys/i386/isa/isa_device.h b/sys/i386/isa/isa_device.h index 3daa6540b427..4f37cd08fa64 100644 --- a/sys/i386/isa/isa_device.h +++ b/sys/i386/isa/isa_device.h @@ -31,12 +31,14 @@ * SUCH DAMAGE. * * from: @(#)isa_device.h 7.1 (Berkeley) 5/9/91 - * $Id$ + * $Id: isa_device.h,v 1.35 1997/02/22 09:36:42 peter Exp $ */ #ifndef _I386_ISA_ISA_DEVICE_H_ #define _I386_ISA_ISA_DEVICE_H_ +#include "opt_smp.h" + /* * ISA Bus Autoconfiguration */ @@ -67,7 +69,11 @@ struct isa_device { int id_id; /* device id */ struct isa_driver *id_driver; int id_iobase; /* base i/o address */ +#if defined(APIC_IO) + u_int id_irq; /* interrupt request */ +#else u_short id_irq; /* interrupt request */ +#endif /* APIC_IO */ short id_drq; /* DMA request */ caddr_t id_maddr; /* physical i/o memory address on bus (if any)*/ int id_msize; /* size of i/o memory */ @@ -142,6 +148,21 @@ inthand_t IDTVEC(intr4), IDTVEC(intr5), IDTVEC(intr6), IDTVEC(intr7), IDTVEC(intr8), IDTVEC(intr9), IDTVEC(intr10), IDTVEC(intr11), IDTVEC(intr12), IDTVEC(intr13), IDTVEC(intr14), IDTVEC(intr15); +#if defined(APIC_IO) +inthand_t + IDTVEC(fastintr16), IDTVEC(fastintr17), + IDTVEC(fastintr18), IDTVEC(fastintr19), + IDTVEC(fastintr20), IDTVEC(fastintr21), + IDTVEC(fastintr22), IDTVEC(fastintr23); +inthand_t + IDTVEC(intr16), IDTVEC(intr17), IDTVEC(intr18), IDTVEC(intr19), + IDTVEC(intr20), IDTVEC(intr21), IDTVEC(intr22), IDTVEC(intr23); +#include <machine/apic.h> +#if defined(IPI_INTS) +inthand_t + IDTVEC(ipi24), IDTVEC(ipi25), IDTVEC(ipi26), IDTVEC(ipi27); +#endif /* IPI_INTS */ +#endif /* APIC_IO */ struct isa_device * find_display __P((void)); struct isa_device * @@ -157,6 +178,9 @@ void isa_dmastart __P((int flags, caddr_t addr, u_int nbytes, int chan)); int isa_dma_acquire __P((int chan)); void isa_dma_release __P((int chan)); int isa_irq_pending __P((struct isa_device *dvp)); +#if defined(APIC_IO) +int icu_irq_pending __P((struct isa_device *dvp)); +#endif /* APIC_IO */ int isa_nmi __P((int cd)); void reconfig_isadev __P((struct isa_device *isdp, u_int *mp)); int register_intr __P((int intr, int device_id, u_int flags, diff --git a/sys/i386/isa/npx.c b/sys/i386/isa/npx.c index 6e3f5c77e077..402b1cebb76f 100644 --- a/sys/i386/isa/npx.c +++ b/sys/i386/isa/npx.c @@ -32,7 +32,7 @@ * SUCH DAMAGE. * * from: @(#)npx.c 7.2 (Berkeley) 5/12/91 - * $Id: npx.c,v 1.40 1997/03/24 11:23:58 bde Exp $ + * $Id: npx.c,v 1.41 1997/04/22 06:55:38 jdp Exp $ */ #include "npx.h" @@ -40,6 +40,7 @@ #include "opt_cpu.h" #include "opt_math_emulate.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -60,6 +61,10 @@ #include <machine/trap.h> #include <machine/clock.h> #include <machine/specialreg.h> +#if defined(APIC_IO) +#include <machine/apic.h> +#include <machine/mpapic.h> +#endif /* APIC_IO */ #include <i386/isa/icu.h> #include <i386/isa/isa_device.h> @@ -133,7 +138,12 @@ SYSCTL_INT(_hw,HW_FLOATINGPT, floatingpoint, "Floatingpoint instructions executed in hardware"); static u_int npx0_imask = SWI_CLOCK_MASK; +#ifdef SMP +#define npxproc (SMPnpxproc[cpunumber()]) +struct proc *SMPnpxproc[NCPU]; +#else struct proc *npxproc; +#endif static bool_t npx_ex16; static bool_t npx_exists; @@ -148,8 +158,28 @@ static volatile u_int npx_traps_while_probing; * interrupts. We'll still need a special exception 16 handler. The busy * latch stuff in probeintr() can be moved to npxprobe(). */ - inthand_t probeintr; + +#if defined(APIC_IO) + +asm +(" + .text + .p2align 2,0x90 +" __XSTRING(CNAME(probeintr)) ": + ss + incl " __XSTRING(CNAME(npx_intrs_while_probing)) " + pushl %eax + movl " __XSTRING(CNAME(apic_base)) ",%eax # EOI to local APIC + movl $0,0xb0(,%eax,1) # movl $0, APIC_EOI(%eax) + movb $0,%al + outb %al,$0xf0 # clear BUSY# latch + popl %eax + iret +"); + +#else + asm (" .text @@ -167,6 +197,8 @@ asm iret "); +#endif /* APIC_IO */ + inthand_t probetrap; asm (" @@ -191,8 +223,12 @@ npxprobe(dvp) { int result; u_long save_eflags; +#if defined(APIC_IO) + u_int save_apic_mask; +#else u_char save_icu1_mask; u_char save_icu2_mask; +#endif /* APIC_IO */ struct gate_descriptor save_idt_npxintr; struct gate_descriptor save_idt_npxtrap; /* @@ -205,20 +241,32 @@ npxprobe(dvp) npx_intrno = NRSVIDT + ffs(dvp->id_irq) - 1; save_eflags = read_eflags(); disable_intr(); +#if defined(APIC_IO) + save_apic_mask = INTRGET(); +#else save_icu1_mask = inb(IO_ICU1 + 1); save_icu2_mask = inb(IO_ICU2 + 1); +#endif /* APIC_IO */ save_idt_npxintr = idt[npx_intrno]; save_idt_npxtrap = idt[16]; +#if defined(APIC_IO) + INTRSET( ~dvp->id_irq ); +#else outb(IO_ICU1 + 1, ~(IRQ_SLAVE | dvp->id_irq)); outb(IO_ICU2 + 1, ~(dvp->id_irq >> 8)); +#endif /* APIC_IO */ setidt(16, probetrap, SDT_SYS386TGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); setidt(npx_intrno, probeintr, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); npx_idt_probeintr = idt[npx_intrno]; enable_intr(); result = npxprobe1(dvp); disable_intr(); +#if defined(APIC_IO) + INTRSET( save_apic_mask ); +#else outb(IO_ICU1 + 1, save_icu1_mask); outb(IO_ICU2 + 1, save_icu2_mask); +#endif /* APIC_IO */ idt[npx_intrno] = save_idt_npxintr; idt[16] = save_idt_npxtrap; write_eflags(save_eflags); @@ -364,7 +412,8 @@ npxattach(dvp) } npxinit(__INITIAL_NPXCW__); -#ifdef I586_CPU +#if defined(I586_CPU) && !defined(SMP) + /* FPU not working under SMP yet */ if (cpu_class == CPUCLASS_586 && npx_ex16) { if (!(dvp->id_flags & NPX_DISABLE_I586_OPTIMIZED_BCOPY)) { bcopy_vector = i586_bcopy; @@ -576,18 +625,32 @@ void npxsave(addr) struct save87 *addr; { +#if defined(APIC_IO) + u_int apic_mask; + u_int old_apic_mask; +#else u_char icu1_mask; u_char icu2_mask; u_char old_icu1_mask; u_char old_icu2_mask; +#endif /* APIC_IO */ struct gate_descriptor save_idt_npxintr; disable_intr(); +#if defined(APIC_IO) + old_apic_mask = INTRGET(); +#else old_icu1_mask = inb(IO_ICU1 + 1); old_icu2_mask = inb(IO_ICU2 + 1); +#endif /* APIC_IO */ save_idt_npxintr = idt[npx_intrno]; +#if defined(APIC_IO) + /** FIXME: try clrIoApicMaskBit( npx0_imask ); */ + INTRSET( old_apic_mask & ~(npx0_imask & 0xffff) ); +#else outb(IO_ICU1 + 1, old_icu1_mask & ~(IRQ_SLAVE | npx0_imask)); outb(IO_ICU2 + 1, old_icu2_mask & ~(npx0_imask >> 8)); +#endif /* APIC_IO */ idt[npx_intrno] = npx_idt_probeintr; enable_intr(); stop_emulating(); @@ -596,6 +659,11 @@ npxsave(addr) start_emulating(); npxproc = NULL; disable_intr(); +#if defined(APIC_IO) + apic_mask = INTRGET(); /* masks may have changed */ + INTRSET( (apic_mask & ~(npx0_imask & 0xffff)) | + (old_apic_mask & (npx0_imask & 0xffff))); +#else icu1_mask = inb(IO_ICU1 + 1); /* masks may have changed */ icu2_mask = inb(IO_ICU2 + 1); outb(IO_ICU1 + 1, @@ -603,6 +671,7 @@ npxsave(addr) outb(IO_ICU2 + 1, (icu2_mask & ~(npx0_imask >> 8)) | (old_icu2_mask & (npx0_imask >> 8))); +#endif /* APIC_IO */ idt[npx_intrno] = save_idt_npxintr; enable_intr(); /* back to usual state */ } diff --git a/sys/i386/isa/random_machdep.c b/sys/i386/isa/random_machdep.c index 64c215c687e3..5fb9ef879c00 100644 --- a/sys/i386/isa/random_machdep.c +++ b/sys/i386/isa/random_machdep.c @@ -1,7 +1,7 @@ /* * random_machdep.c -- A strong random number generator * - * $Id$ + * $Id: random_machdep.c,v 1.15 1997/02/22 09:37:02 peter Exp $ * * Version 0.95, last modified 18-Oct-95 * @@ -192,7 +192,7 @@ add_timer_randomness(struct random_bucket *r, struct timer_rand_state *state, u_int nbits; u_int32_t time; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) if (i586_ctr_freq != 0) { num ^= (u_int32_t) rdtsc() << 16; r->entropy_count += 2; @@ -204,7 +204,7 @@ add_timer_randomness(struct random_bucket *r, struct timer_rand_state *state, num ^= inb(TIMER_CNTR0) << 24; enable_intr(); r->entropy_count += 2; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) } #endif diff --git a/sys/i386/isa/sio.c b/sys/i386/isa/sio.c index 7d5e78db1be2..54420e863661 100644 --- a/sys/i386/isa/sio.c +++ b/sys/i386/isa/sio.c @@ -31,12 +31,13 @@ * SUCH DAMAGE. * * from: @(#)com.c 7.5 (Berkeley) 5/16/91 - * $Id: sio.c,v 1.162 1997/04/05 13:11:27 bde Exp $ + * $Id: sio.c,v 1.163 1997/04/18 18:28:09 bde Exp $ */ #include "opt_comconsole.h" #include "opt_ddb.h" #include "opt_sio.h" +#include "opt_smp.h" #include "sio.h" /* @@ -82,6 +83,16 @@ #include <pccard/slot.h> #endif +#if defined(APIC_IO) +/* + * INTs are masked in the (global) IO APIC, + * but the IRR register is in each LOCAL APIC, + * so we HAVE to unmask the INT to be able to "see INT pending" + * BUT how do we clear them??? + */ +#define isa_irq_pending icu_irq_pending +#endif /* APIC_IO */ + #define LOTS_OF_EVENTS 64 /* helps separate urgent events from input */ #define RB_I_HIGH_WATER (TTYHOG - 2 * RS_IBUFSIZE) #define RS_IBUFSIZE 256 diff --git a/sys/i386/isa/vector.s b/sys/i386/isa/vector.s index fa2fd42afb54..270ae208cc29 100644 --- a/sys/i386/isa/vector.s +++ b/sys/i386/isa/vector.s @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id$ + * $Id: vector.s,v 1.25 1997/02/22 09:37:23 peter Exp $ */ /* @@ -8,6 +8,12 @@ */ #include "opt_auto_eoi.h" +#include "opt_smp.h" + +#if defined(SMP) +#include <machine/smpasm.h> /* this includes <machine/apic.h> */ +#include <machine/smptests.h> /** TEST_CPUHITS */ +#endif /* SMP */ #include <i386/isa/icu.h> #ifdef PC98 @@ -22,6 +28,111 @@ #define ICU_IMR_OFFSET 1 /* IO_ICU{1,2} + 1 */ #endif + +#if defined(SMP) + +#define GET_MPLOCK call _get_mplock +#define REL_MPLOCK call _rel_mplock + +#else + +#define GET_MPLOCK /* NOP get Kernel Mutex */ +#define REL_MPLOCK /* NOP release mutex */ + +#endif /* SMP */ + + +#if defined(APIC_IO) + +#if defined(SMP) && defined(TEST_CPUHITS) + +#undef GET_MPLOCK +#define GET_MPLOCK \ + call _get_mplock ; \ + GETCPUID(%eax) ; \ + incl _cpuhits(,%eax,4) + +#endif /* SMP && TEST_CPUHITS */ + +#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) +#define IRQ_BIT(irq_num) (1 << (irq_num)) + +#define ENABLE_APIC \ + movl _apic_base, %eax ; \ + movl $0, APIC_EOI(%eax) + +#define ENABLE_ICU1 ENABLE_APIC +#define ENABLE_ICU1_AND_2 ENABLE_APIC + +#define MASK_IRQ(irq_num,icu) \ + orl $IRQ_BIT(irq_num),_imen ; /* set the mask bit */ \ + movl _io_apic_base,%ecx ; /* io apic addr */ \ + movl $REDTBL_IDX(irq_num),(%ecx) ; /* write the index */ \ + movl IOAPIC_WINDOW(%ecx),%eax ; /* current value */ \ + orl $IOART_INTMASK,%eax ; /* set the mask */ \ + movl %eax,IOAPIC_WINDOW(%ecx) ; /* new value */ + +#define UNMASK_IRQ(irq_num,icu) \ + andl $~IRQ_BIT(irq_num),_imen ; /* clear mask bit */ \ + movl _io_apic_base,%ecx ; /* io apic addr */ \ + movl $REDTBL_IDX(irq_num),(%ecx) ; /* write the index */ \ + movl IOAPIC_WINDOW(%ecx),%eax ; /* current value */ \ + andl $~IOART_INTMASK,%eax ; /* clear the mask */ \ + movl %eax,IOAPIC_WINDOW(%ecx) ; /* new value */ + +#define TEST_IRQ(irq_num,reg) \ + testl $IRQ_BIT(irq_num),%eax + +#define SET_IPENDING(irq_num) \ + orl $IRQ_BIT(irq_num),_ipending + +/* + * 'lazy masking' code submitted by: Bruce Evans <bde@zeta.org.au> + */ +#define MAYBE_MASK_IRQ(irq_num,icu) \ + testl $IRQ_BIT(irq_num),iactive ; /* lazy masking */ \ + je 1f ; /* NOT currently active */ \ + MASK_IRQ(irq_num,icu) ; \ + ENABLE_APIC ; \ + SET_IPENDING(irq_num) ; \ + REL_MPLOCK ; /* SMP release global lock */ \ + popl %es ; \ + popl %ds ; \ + popal ; \ + addl $4+4,%esp ; \ + iret ; \ +; \ + ALIGN_TEXT ; \ +1: ; \ + orl $IRQ_BIT(irq_num),iactive + +#define MAYBE_UNMASK_IRQ(irq_num,icu) \ + andl $~IRQ_BIT(irq_num),iactive ; \ + testl $IRQ_BIT(irq_num),_imen ; \ + je 3f ; \ + UNMASK_IRQ(irq_num,icu) ; \ +3: + +#else /* APIC_IO */ + +#define MASK_IRQ(irq_num,icu) \ + movb _imen + IRQ_BYTE(irq_num),%al ; \ + orb $IRQ_BIT(irq_num),%al ; \ + movb %al,_imen + IRQ_BYTE(irq_num) ; \ + outb %al,$icu+ICU_IMR_OFFSET + +#define UNMASK_IRQ(irq_num,icu) \ + movb _imen + IRQ_BYTE(irq_num),%al ; \ + andb $~IRQ_BIT(irq_num),%al ; \ + movb %al,_imen + IRQ_BYTE(irq_num) ; \ + outb %al,$icu+ICU_IMR_OFFSET + +#define TEST_IRQ(irq_num,reg) \ + testb $IRQ_BIT(irq_num),%reg + +#define SET_IPENDING(irq_num) \ + orb $IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num) + #define ICU_EOI 0x20 /* XXX - define elsewhere */ #define IRQ_BIT(irq_num) (1 << ((irq_num) % 8)) @@ -50,6 +161,15 @@ OUTB_ICU1 /* ... then first icu (if !AUTO_EOI_1) */ #endif +#define MAYBE_MASK_IRQ(irq_num,icu) \ + MASK_IRQ(irq_num,icu) + +#define MAYBE_UNMASK_IRQ(irq_num,icu) \ + UNMASK_IRQ(irq_num,icu) + +#endif /* APIC_IO */ + + #ifdef FAST_INTR_HANDLER_USES_ES #define ACTUALLY_PUSHED 1 #define MAYBE_MOVW_AX_ES movl %ax,%es @@ -125,6 +245,7 @@ IDTVEC(vec_name) ; \ movl %ax,%ds ; \ MAYBE_MOVW_AX_ES ; \ FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \ + GET_MPLOCK ; /* SMP Spin lock */ \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -138,6 +259,7 @@ IDTVEC(vec_name) ; \ jne 2f ; /* yes, maybe handle them */ \ 1: ; \ MEXITCOUNT ; \ + REL_MPLOCK ; /* SMP release global lock */ \ MAYBE_POPL_ES ; \ popl %ds ; \ popl %edx ; \ @@ -183,13 +305,11 @@ IDTVEC(vec_name) ; \ movl $KDSEL,%eax ; /* ... and reload with kernel's own ... */ \ movl %ax,%ds ; /* ... early for obsolete reasons */ \ movl %ax,%es ; \ - movb _imen + IRQ_BYTE(irq_num),%al ; \ - orb $IRQ_BIT(irq_num),%al ; \ - movb %al,_imen + IRQ_BYTE(irq_num) ; \ - outb %al,$icu+ICU_IMR_OFFSET ; \ + GET_MPLOCK ; /* SMP Spin lock */ \ + MAYBE_MASK_IRQ(irq_num,icu) ; \ enable_icus ; \ movl _cpl,%eax ; \ - testb $IRQ_BIT(irq_num),%reg ; \ + TEST_IRQ(irq_num,reg) ; \ jne 2f ; \ incb _intr_nesting_level ; \ __CONCAT(Xresume,irq_num): ; \ @@ -205,10 +325,7 @@ __CONCAT(Xresume,irq_num): ; \ sti ; \ call *_intr_handler + (irq_num) * 4 ; \ cli ; /* must unmask _imen and icu atomically */ \ - movb _imen + IRQ_BYTE(irq_num),%al ; \ - andb $~IRQ_BIT(irq_num),%al ; \ - movb %al,_imen + IRQ_BYTE(irq_num) ; \ - outb %al,$icu+ICU_IMR_OFFSET ; \ + MAYBE_UNMASK_IRQ(irq_num,icu) ; \ sti ; /* XXX _doreti repeats the cli/sti */ \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ @@ -218,13 +335,66 @@ __CONCAT(Xresume,irq_num): ; \ ALIGN_TEXT ; \ 2: ; \ /* XXX skip mcounting here to avoid double count */ \ - orb $IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num) ; \ + SET_IPENDING(irq_num) ; \ + REL_MPLOCK ; /* SMP release global lock */ \ popl %es ; \ popl %ds ; \ popal ; \ addl $4+4,%esp ; \ iret + +#if defined(APIC_IO) && defined(IPI_INTS) +/* + * A simple IPI_INTR() macro based on a heavily cut down FAST_INTR(). + * call it's handler, EOI and return. + */ +#define IPI_INTR(irq_num, vec_name) \ + .text ; \ + SUPERALIGN_TEXT ; \ +IDTVEC(vec_name) ; \ + pushl %eax ; /* save only call-used registers */ \ + pushl %ecx ; \ + pushl %edx ; \ + pushl %ds ; \ + MAYBE_PUSHL_ES ; \ + movl $KDSEL,%eax ; \ + movl %ax,%ds ; \ + MAYBE_MOVW_AX_ES ; \ + pushl _intr_unit + (irq_num) * 4 ; \ + call *_intr_handler + (irq_num) * 4 ; \ + ENABLE_APIC ; \ + addl $4,%esp ; \ + incl _cnt+V_INTR ; /* book-keeping can wait */ \ + movl _intr_countp + (irq_num) * 4,%eax ; \ + incl (%eax) ; \ + MAYBE_POPL_ES ; \ + popl %ds ; \ + popl %edx ; \ + popl %ecx ; \ + popl %eax ; \ + iret +#endif /* APIC_IO && IPI_INTS */ + +#if defined(XFAST_IPI32) + .text + SUPERALIGN_TEXT + .globl _Xfastipi32 +_Xfastipi32: + pushl %eax + movl %cr3, %eax + movl %eax, %cr3 + pushl %ds + movl $KDSEL,%eax + movl %ax,%ds + incl _ipihits + movl _apic_base, %eax + movl $0, APIC_EOI(%eax) + popl %ds + popl %eax + iret +#endif /* XFAST_IPI32 */ + MCOUNT_LABEL(bintr) FAST_INTR(0,fastintr0, ENABLE_ICU1) FAST_INTR(1,fastintr1, ENABLE_ICU1) @@ -242,6 +412,16 @@ MCOUNT_LABEL(bintr) FAST_INTR(13,fastintr13, ENABLE_ICU1_AND_2) FAST_INTR(14,fastintr14, ENABLE_ICU1_AND_2) FAST_INTR(15,fastintr15, ENABLE_ICU1_AND_2) +#if defined(APIC_IO) + FAST_INTR(16,fastintr16, ENABLE_ICU1_AND_2) + FAST_INTR(17,fastintr17, ENABLE_ICU1_AND_2) + FAST_INTR(18,fastintr18, ENABLE_ICU1_AND_2) + FAST_INTR(19,fastintr19, ENABLE_ICU1_AND_2) + FAST_INTR(20,fastintr20, ENABLE_ICU1_AND_2) + FAST_INTR(21,fastintr21, ENABLE_ICU1_AND_2) + FAST_INTR(22,fastintr22, ENABLE_ICU1_AND_2) + FAST_INTR(23,fastintr23, ENABLE_ICU1_AND_2) +#endif /* APIC_IO */ INTR(0,intr0, IO_ICU1, ENABLE_ICU1, al) INTR(1,intr1, IO_ICU1, ENABLE_ICU1, al) INTR(2,intr2, IO_ICU1, ENABLE_ICU1, al) @@ -258,6 +438,22 @@ MCOUNT_LABEL(bintr) INTR(13,intr13, IO_ICU2, ENABLE_ICU1_AND_2, ah) INTR(14,intr14, IO_ICU2, ENABLE_ICU1_AND_2, ah) INTR(15,intr15, IO_ICU2, ENABLE_ICU1_AND_2, ah) +#if defined(APIC_IO) + INTR(16,intr16, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(17,intr17, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(18,intr18, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(19,intr19, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(20,intr20, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(21,intr21, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(22,intr22, IO_ICU2, ENABLE_ICU1_AND_2, ah) + INTR(23,intr23, IO_ICU2, ENABLE_ICU1_AND_2, ah) +#if defined(IPI_INTS) + IPI_INTR(24, ipi24) + IPI_INTR(25, ipi25) + IPI_INTR(26, ipi26) + IPI_INTR(27, ipi27) +#endif /* IPI_INTS */ +#endif /* APIC_IO */ MCOUNT_LABEL(eintr) .data @@ -267,17 +463,74 @@ ihandlers: /* addresses of interrupt handlers */ .long Xresume4, Xresume5, Xresume6, Xresume7 .long Xresume8, Xresume9, Xresume10, Xresume11 .long Xresume12, Xresume13, Xresume14, Xresume15 +#if defined(APIC_IO) + .long Xresume16, Xresume17, Xresume18, Xresume19 + .long Xresume20, Xresume21, Xresume22, Xresume23 + .long 0, 0, 0, 0, swi_tty, swi_net, _softclock, swi_ast +#else .long swi_tty, swi_net, 0, 0, 0, 0, 0, 0 .long 0, 0, 0, 0, 0, 0, _softclock, swi_ast +#endif /* APIC_IO */ + imasks: /* masks for interrupt handlers */ .space NHWI*4 /* padding; HWI masks are elsewhere */ + +#if defined(APIC_IO) +#if defined(IPI_INTS) + /* these 4 IPI slots are counted as HARDWARE INTs, ie NHWI, above */ +#else + .long 0, 0, 0, 0 /* padding */ +#endif /* IPI_INTS */ + .long SWI_TTY_MASK, SWI_NET_MASK, SWI_CLOCK_MASK, SWI_AST_MASK +#else .long SWI_TTY_MASK, SWI_NET_MASK, 0, 0, 0, 0, 0, 0 .long 0, 0, 0, 0, 0, 0, SWI_CLOCK_MASK, SWI_AST_MASK +#endif /* APIC_IO */ + .globl _intr_nesting_level _intr_nesting_level: .byte 0 .space 3 +#if defined(APIC_IO) + + .globl _ivectors +_ivectors: + .long _Xintr0, _Xintr1, _Xintr2, _Xintr3 + .long _Xintr4, _Xintr5, _Xintr6, _Xintr7 + .long _Xintr8, _Xintr9, _Xintr10, _Xintr11 + .long _Xintr12, _Xintr13, _Xintr14, _Xintr15 + .long _Xintr16, _Xintr17, _Xintr18, _Xintr19 + .long _Xintr20, _Xintr21, _Xintr22, _Xintr23 +#if defined(IPI_INTS) + .long _Xipi24, _Xipi25, _Xipi26, _Xipi27 +#endif /* IPI_INTS */ + +/* active flag for lazy masking */ +iactive: + .long 0 + +#if defined(XFAST_IPI32) + .globl _ipihits +_ipihits: + .long 0 +#endif /* XFAST_IPI32 */ + +#if defined(TEST_CPUHITS) + .globl _cpuhits +_cpuhits: +#if !defined(NCPU) +/** + * FIXME: need a way to pass NCPU to .s files. + * NCPU currently defined in smp.h IF NOT defined in opt_smp.h. + */ +#define NCPU 4 +#endif /* NCPU */ + .space NCPU*4 +#endif /* TEST_CPUHITS */ + +#endif /* APIC_IO */ + /* * Interrupt counters and names. The format of these and the label names * must agree with what vmstat expects. The tables are indexed by device @@ -309,6 +562,22 @@ _intrnames: .asciz "stray irq13" .asciz "stray irq14" .asciz "stray irq15" +#if defined(APIC_IO) + .asciz "stray irq16" + .asciz "stray irq17" + .asciz "stray irq18" + .asciz "stray irq19" + .asciz "stray irq20" + .asciz "stray irq21" + .asciz "stray irq22" + .asciz "stray irq23" +#if defined(IPI_INTS) + .asciz "stray irq24" + .asciz "stray irq25" + .asciz "stray irq26" + .asciz "stray irq27" +#endif /* IPI_INTS */ +#endif /* APIC_IO */ _eintrnames: .text diff --git a/sys/isa/atrtc.c b/sys/isa/atrtc.c index b9425536d67a..d46f982806bd 100644 --- a/sys/isa/atrtc.c +++ b/sys/isa/atrtc.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)clock.c 7.2 (Berkeley) 5/12/91 - * $Id: clock.c,v 1.79 1997/03/05 08:08:48 bde Exp $ + * $Id: clock.c,v 1.80 1997/04/06 13:25:48 mckay Exp $ */ /* @@ -49,6 +49,7 @@ */ #include "opt_clock.h" +#include "opt_smp.h" #include "opt_cpu.h" #include <sys/param.h> @@ -100,7 +101,7 @@ int adjkerntz; /* local offset from GMT in seconds */ int disable_rtc_set; /* disable resettodr() if != 0 */ u_int idelayed; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) u_int i586_ctr_bias; u_int i586_ctr_comultiplier; u_int i586_ctr_freq; @@ -143,7 +144,7 @@ static u_char timer0_state; static u_char timer2_state; static void (*timer_func) __P((struct clockframe *frame)) = hardclock; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) static void set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq); #endif static void set_timer_freq(u_int freq, int intr_freq); @@ -553,7 +554,7 @@ calibrate_clocks(void) goto fail; tot_count = 0; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) if (cpu_class == CPUCLASS_586 || cpu_class == CPUCLASS_686) wrmsr(0x10, 0LL); /* XXX 0x10 is the MSR for the TSC */ #endif @@ -586,7 +587,7 @@ calibrate_clocks(void) goto fail; } -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * Read the cpu cycle counter. The timing considerations are * similar to those for the i8254 clock. @@ -667,14 +668,14 @@ startrtclock() printf( "%d Hz differs from default of %d Hz by more than 1%%\n", freq, timer_freq); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) i586_ctr_freq = 0; #endif } set_timer_freq(timer_freq, hz); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) #ifndef CLK_USE_I586_CALIBRATION if (i586_ctr_freq != 0) { if (bootverbose) @@ -821,6 +822,20 @@ resettodr() writertc(RTC_STATUSB, rtc_statusb); } +#if defined(APIC_IO) + +/* from icu.s: */ +extern u_int hwisrs[]; +extern void vec8254 __P((void)); +extern void vecRTC __P((void)); +extern u_int ivectors[]; +extern u_int Xintr8254; +extern u_int XintrRTC; +extern u_int mask8254; +extern u_int maskRTC; + +#endif /* APIC_IO */ + /* * Start both clocks running. */ @@ -828,6 +843,9 @@ void cpu_initclocks() { int diag; +#if defined(APIC_IO) + int x; +#endif /* APIC_IO */ if (statclock_disable) { /* @@ -845,11 +863,47 @@ cpu_initclocks() } /* Finish initializing 8253 timer 0. */ +#if defined(APIC_IO) + /* 8254 is traditionally on ISA IRQ0 */ + if ((x = get_isa_apic_irq(0)) < 0) { + /* + * bummer, this mb doesn't have the 8254 on ISA irq0, + * perhaps it's on the EISA bus... + */ + if ((x = get_eisa_apic_irq(0)) < 0) { + /* double bummer, attempt to redirect thru the 8259 */ + if (bootverbose) + printf("APIC missing 8254 connection\n"); + + /* allow 8254 timer to INTerrupt 8259 */ +#if !defined(IO_ICU1) +#define IO_ICU1 0x20 +#endif + x = inb(IO_ICU1 + 1); /* current mask in 8259 */ + x &= ~1; /* clear 8254 timer mask */ + outb(IO_ICU1 + 1, x); /* write new mask */ + + /* program IO APIC for type 3 INT on INT0 */ + if (ext_int_setup(0, 0) < 0) + panic("8254 redirect impossible!"); + x = 0; /* 8259 is on 0 */ + } + } + + hwisrs[x] = (u_int)vec8254; + Xintr8254 = (u_int)ivectors[x]; /* XXX might need Xfastintr# */ + mask8254 = (1 << x); + register_intr(/* irq */ x, /* XXX id */ 0, /* flags */ 0, + /* XXX */ (inthand2_t *)clkintr, &clk_imask, + /* unit */ 0); + INTREN(mask8254); +#else register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0, /* XXX */ (inthand2_t *)clkintr, &clk_imask, /* unit */ 0); INTREN(IRQ0); -#if defined(I586_CPU) || defined(I686_CPU) +#endif /* APIC_IO */ +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) /* * Finish setting up anti-jitter measures. */ @@ -867,11 +921,36 @@ cpu_initclocks() diag = rtcin(RTC_DIAG); if (diag != 0) printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS); +#if defined(APIC_IO) + /* RTC is traditionally on ISA IRQ8 */ + if ((x = get_isa_apic_irq(8)) < 0) { + if ((x = get_eisa_apic_irq(8)) < 0) { + panic("APIC missing RTC connection"); + } + } + + hwisrs[x] = (u_int)vecRTC; + XintrRTC = (u_int)ivectors[x]; /* XXX might need Xfastintr# */ + maskRTC = (1 << x); + register_intr(/* irq */ x, /* XXX id */ 1, /* flags */ 0, + /* XXX */ (inthand2_t *)rtcintr, &stat_imask, + /* unit */ 0); + INTREN(maskRTC); +#else register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0, /* XXX */ (inthand2_t *)rtcintr, &stat_imask, /* unit */ 0); INTREN(IRQ8); +#endif /* APIC_IO */ writertc(RTC_STATUSB, rtc_statusb); + +#if defined(APIC_IO) + printf("Enabled INTs: "); + for (x = 0; x < 24; ++x) + if ((imen & (1 << x)) == 0) + printf("%d, ", x); + printf("imen: 0x%08x\n", imen); +#endif /* APIC_IO */ } void @@ -900,7 +979,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS if (timer0_state != 0) return (EBUSY); /* too much trouble to handle */ set_timer_freq(freq, hz); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) set_i586_ctr_freq(i586_ctr_freq, timer_freq); #endif } @@ -910,7 +989,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS SYSCTL_PROC(_machdep, OID_AUTO, i8254_freq, CTLTYPE_INT | CTLFLAG_RW, 0, sizeof(u_int), sysctl_machdep_i8254_freq, "I", ""); -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) static void set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq) { @@ -949,4 +1028,4 @@ sysctl_machdep_i586_freq SYSCTL_HANDLER_ARGS SYSCTL_PROC(_machdep, OID_AUTO, i586_freq, CTLTYPE_INT | CTLFLAG_RW, 0, sizeof(u_int), sysctl_machdep_i586_freq, "I", ""); -#endif /* defined(I586_CPU) || defined(I686_CPU) */ +#endif /* (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) */ diff --git a/sys/isa/sio.c b/sys/isa/sio.c index 7d5e78db1be2..54420e863661 100644 --- a/sys/isa/sio.c +++ b/sys/isa/sio.c @@ -31,12 +31,13 @@ * SUCH DAMAGE. * * from: @(#)com.c 7.5 (Berkeley) 5/16/91 - * $Id: sio.c,v 1.162 1997/04/05 13:11:27 bde Exp $ + * $Id: sio.c,v 1.163 1997/04/18 18:28:09 bde Exp $ */ #include "opt_comconsole.h" #include "opt_ddb.h" #include "opt_sio.h" +#include "opt_smp.h" #include "sio.h" /* @@ -82,6 +83,16 @@ #include <pccard/slot.h> #endif +#if defined(APIC_IO) +/* + * INTs are masked in the (global) IO APIC, + * but the IRR register is in each LOCAL APIC, + * so we HAVE to unmask the INT to be able to "see INT pending" + * BUT how do we clear them??? + */ +#define isa_irq_pending icu_irq_pending +#endif /* APIC_IO */ + #define LOTS_OF_EVENTS 64 /* helps separate urgent events from input */ #define RB_I_HIGH_WATER (TTYHOG - 2 * RS_IBUFSIZE) #define RS_IBUFSIZE 256 diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index f264c845a63a..4630283305d8 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -39,10 +39,11 @@ * SUCH DAMAGE. * * @(#)init_main.c 8.9 (Berkeley) 1/21/94 - * $Id: init_main.c,v 1.59 1997/03/22 06:52:55 bde Exp $ + * $Id: init_main.c,v 1.60 1997/04/07 07:15:59 peter Exp $ */ #include "opt_rlimit.h" +#include "opt_smp.h" #include "opt_devfs.h" #include <sys/param.h> @@ -62,6 +63,9 @@ #include <sys/vmmeter.h> #include <machine/cpu.h> +#if defined(SMP) +#include <machine/smp.h> +#endif /* SMP */ #include <vm/vm.h> #include <vm/vm_param.h> @@ -76,6 +80,7 @@ extern struct linker_set sysinit_set; /* XXX */ extern void __main __P((void)); extern void main __P((void *framep)); +extern void secondary_main __P((void)); /* Components of the first process -- never freed. */ static struct session session0; @@ -85,7 +90,9 @@ static struct pcred cred0; static struct filedesc0 filedesc0; static struct plimit limit0; static struct vmspace vmspace0; +#ifndef SMP /* per-cpu on smp */ struct proc *curproc = &proc0; +#endif struct proc *initproc; int cmask = CMASK; @@ -98,7 +105,13 @@ struct timeval boottime; SYSCTL_STRUCT(_kern, KERN_BOOTTIME, boottime, CTLFLAG_RW, &boottime, timeval, ""); +/* + * for SMP, the runtime variable has to be per-cpu, so we use the + * extern declaration in sys/kernel.h + */ +#ifndef SMP struct timeval runtime; +#endif /* * Promiscuous argument pass for start_init() @@ -198,6 +211,7 @@ main(framep) } } + panic("Shouldn't get here!"); /* NOTREACHED*/ } @@ -216,6 +230,11 @@ kproc_start(udata) struct kproc_desc *kp = udata; struct proc *p = curproc; +#ifdef DIAGNOSTIC + printf("Start pid=%d <%s>\n",p->p_pid, kp->arg0); +#endif + + /* save a global descriptor, if desired*/ if( kp->global_procpp != NULL) *kp->global_procpp = p; diff --git a/sys/kern/init_smp.c b/sys/kern/init_smp.c new file mode 100644 index 000000000000..b7ed538c24e6 --- /dev/null +++ b/sys/kern/init_smp.c @@ -0,0 +1,298 @@ +/* + * Copyright (c) 1996, Peter Wemm <peter@freebsd.org> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: init_smp.c,v 1.46 1997/04/25 03:10:41 fsmp Exp $ + */ + +#include "opt_smp.h" +#include "opt_smp_autostart.h" +#include "opt_smp_invltlb.h" + +#include <sys/param.h> +#include <sys/filedesc.h> +#include <sys/kernel.h> +#include <sys/sysctl.h> +#include <sys/proc.h> +#include <sys/resourcevar.h> +#include <sys/signalvar.h> +#include <sys/systm.h> +#include <sys/vnode.h> +#include <sys/sysent.h> +#include <sys/reboot.h> +#include <sys/sysproto.h> +#include <sys/vmmeter.h> +#include <sys/lock.h> + +#include <machine/cpu.h> +#include <machine/smp.h> +#include <machine/smptests.h> /** IGNORE_IDLEPROCS */ + +#include <vm/vm.h> +#include <vm/vm_param.h> +#include <vm/vm_prot.h> +#include <vm/pmap.h> +#include <vm/vm_map.h> +#include <sys/user.h> + + +int smp_active = 0; /* is the secondary allowed to run? */ +SYSCTL_INT(_kern, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, ""); + +int smp_cpus = 0; /* how many cpu's running */ +SYSCTL_INT(_kern, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, ""); + +int idle_debug = 0; +SYSCTL_INT(_kern, OID_AUTO, idle_debug, CTLFLAG_RW, &idle_debug, 0, ""); + +#if defined(SMP_INVLTLB) +int invldebug = 0; /* XXX: see bit field definitions where used */ +SYSCTL_INT(_kern, OID_AUTO, invldebug, CTLFLAG_RW, &invldebug, 0, ""); +#endif + +#if defined(IGNORE_IDLEPROCS) +int ignore_idleprocs = 1; +#else +int ignore_idleprocs = 0; +#endif +SYSCTL_INT(_kern, OID_AUTO, ignore_idleprocs, CTLFLAG_RW, &ignore_idleprocs, + 0, ""); + +static void smp_kickoff __P((void *dummy)); +SYSINIT(smpkick, SI_SUB_SMP, SI_ORDER_FIRST, smp_kickoff, NULL) + +static void smp_idleloop __P((void *)); + +void secondary_main __P((void)); + +static int idle_loops = 0; +void boot_unlock __P((void)); + +struct proc *SMPidleproc[NCPU]; +static int cpu_starting = -1; + +static void +smp_kickoff(dummy) + void *dummy; +{ + int rval[2]; /* return from fork */ + struct proc *p; + int i; + + /* + * Create the appropriate number of cpu-idle-eaters + */ + for (i = 0; i < mp_ncpus; i++) { + /* kernel thread*/ + if (fork(&proc0, NULL, rval)) + panic("cannot fork idle process"); + p = pfind(rval[0]); + cpu_set_fork_handler(p, smp_idleloop, NULL); + SMPidleproc[i] = p; + p->p_flag |= P_INMEM | P_SYSTEM | P_IDLEPROC; + sprintf(p->p_comm, "cpuidle%d", i); + + /* + * PRIO_IDLE is the last scheduled of the three + * classes and we choose the lowest priority possible + * for there. + */ + p->p_rtprio.type = RTP_PRIO_IDLE; + p->p_rtprio.prio = RTP_PRIO_MAX; + } + +} + + +#define MSG_CPU_MADEIT \ + printf("SMP: TADA! CPU #%d made it into the scheduler!.\n", \ + cpunumber()) +#define MSG_NEXT_CPU \ + printf("SMP: %d of %d CPU's online. Unlocking next CPU..\n", \ + smp_cpus, mp_ncpus) +#define MSG_FINAL_CPU \ + printf("SMP: All %d CPU's are online!\n", \ + smp_cpus) +#define MSG_TOOMANY_CPU \ + printf("SMP: Hey! Too many cpu's started, %d of %d running!\n", \ + smp_cpus, mp_ncpus) + +/* + * This is run by the secondary processor to kick things off. + * It basically drops into the switch routine to pick the first + * available process to run, which is probably an idle process. + */ + +void +secondary_main() +{ + get_mplock(); + + /* + * Record our ID so we know when we've released the mp_stk. + * We must remain single threaded through this. + */ + cpu_starting = cpunumber(); + smp_cpus++; + + printf("SMP: AP CPU #%d LAUNCHED!! Starting Scheduling...\n", + cpunumber()); + + curproc = NULL; /* ensure no context to save */ + cpu_switch(curproc); /* start first process */ + panic("switch returned!"); +} + + +/* + * The main program loop for the idle process + */ + +static void +smp_idleloop(dummy) +void *dummy; +{ + int dcnt = 0; + + /* + * This code is executed only on startup of the idleprocs + * The fact that this is executed is an indication that the + * idle procs are online and it's safe to kick off the first + * AP cpu. + */ + if ( ++idle_loops == mp_ncpus ) { + printf("SMP: All idle procs online.\n"); + +#if defined(SMP_AUTOSTART) +#error WARNING: this code is broken, remove this line at your own risk! + printf("SMP: Starting 1st AP!\n"); + smp_cpus = 1; + smp_active = mp_ncpus; /* XXX */ + boot_unlock(); +#endif /* SMP_AUTOSTART */ + } + + spl0(); + rel_mplock(); + + while (1) { + /* + * make the optimiser assume nothing about the + * which*qs variables + */ + __asm __volatile("" : : : "memory"); + +#if !defined(SMP_AUTOSTART) + /* + * Alternate code to enable a lockstep startup + * via sysctl instead of automatically. + */ + if (smp_cpus == 0 && smp_active != 0) { + get_mplock(); + printf("SMP: Starting 1st AP!\n"); + smp_cpus = 1; + smp_active = mp_ncpus; /* XXX */ + boot_unlock(); + rel_mplock(); + } +#endif /* !SMP_AUTOSTART */ + + /* + * If smp_active is set to (say) 1, we want cpu id's + * 1,2,etc to freeze here. + */ + if (smp_active && smp_active <= cpunumber()) { + get_mplock(); + printf("SMP: cpu#%d freezing\n", cpunumber()); + wakeup((caddr_t)&smp_active); + rel_mplock(); + + while (smp_active <= cpunumber()) { + __asm __volatile("" : : : "memory"); + } + get_mplock(); + printf("SMP: cpu#%d waking up!\n", cpunumber()); + rel_mplock(); + } + + if (whichqs || whichrtqs || (!ignore_idleprocs && whichidqs)) { + /* grab lock for kernel "entry" */ + get_mplock(); + + /* We need to retest due to the spin lock */ + __asm __volatile("" : : : "memory"); + + if (whichqs || whichrtqs || + (!ignore_idleprocs && whichidqs)) { + splhigh(); + if (curproc) + setrunqueue(curproc); + cnt.v_swtch++; + cpu_switch(curproc); + microtime(&runtime); + + if (cpu_starting != -1 && + cpu_starting == cpunumber()) { + /* + * TADA! we have arrived! unlock the + * next cpu now that we have released + * the single mp_stk. + */ + MSG_CPU_MADEIT; + cpu_starting = -1; + + if (smp_cpus < mp_ncpus) { + MSG_NEXT_CPU; + boot_unlock(); + } else if (smp_cpus > mp_ncpus) { + MSG_TOOMANY_CPU; + panic("too many cpus"); + } else { + MSG_FINAL_CPU; +#if defined(SMP_INVLTLB) + /* + * It's safe to send IPI's now + * that all CPUs are online. + */ + invldebug = 6; +#endif + } + + /* Init local apic for irq's */ + apic_initialize(0); + } + + (void)spl0(); + } + rel_mplock(); + } else { + dcnt++; + if (idle_debug && (dcnt % idle_debug) == 0) { + get_mplock(); + printf("idleproc pid#%d on cpu#%d, lock %08x\n", + curproc->p_pid, cpunumber(), mp_lock); + rel_mplock(); + } + } + } +} diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c index 171ed0e98a45..7cbdac46ea3b 100644 --- a/sys/kern/kern_clock.c +++ b/sys/kern/kern_clock.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * @(#)kern_clock.c 8.5 (Berkeley) 1/21/94 - * $Id: kern_clock.c,v 1.34 1997/03/22 16:52:19 mpp Exp $ + * $Id: kern_clock.c,v 1.35 1997/03/22 20:34:21 mpp Exp $ */ /* Portions of this software are covered by the following: */ @@ -978,7 +978,7 @@ statclock(frame) if (p != NULL) p->p_iticks++; cp_time[CP_INTR]++; - } else if (p != NULL) { + } else if (p != NULL && !(p->p_flag & P_IDLEPROC)) { p->p_sticks++; cp_time[CP_SYS]++; } else diff --git a/sys/kern/kern_random.c b/sys/kern/kern_random.c index 64c215c687e3..5fb9ef879c00 100644 --- a/sys/kern/kern_random.c +++ b/sys/kern/kern_random.c @@ -1,7 +1,7 @@ /* * random_machdep.c -- A strong random number generator * - * $Id$ + * $Id: random_machdep.c,v 1.15 1997/02/22 09:37:02 peter Exp $ * * Version 0.95, last modified 18-Oct-95 * @@ -192,7 +192,7 @@ add_timer_randomness(struct random_bucket *r, struct timer_rand_state *state, u_int nbits; u_int32_t time; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) if (i586_ctr_freq != 0) { num ^= (u_int32_t) rdtsc() << 16; r->entropy_count += 2; @@ -204,7 +204,7 @@ add_timer_randomness(struct random_bucket *r, struct timer_rand_state *state, num ^= inb(TIMER_CNTR0) << 24; enable_intr(); r->entropy_count += 2; -#if defined(I586_CPU) || defined(I686_CPU) +#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) } #endif diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c index fe50cf913be3..58bdd393e701 100644 --- a/sys/kern/kern_resource.c +++ b/sys/kern/kern_resource.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * @(#)kern_resource.c 8.5 (Berkeley) 1/21/94 - * $Id$ + * $Id: kern_resource.c,v 1.24 1997/02/22 09:39:09 peter Exp $ */ #include "opt_rlimit.h" @@ -498,7 +498,7 @@ calcru(p, up, sp, ip) sec = p->p_rtime.tv_sec; usec = p->p_rtime.tv_usec; - if (p == curproc) { + if (p == curproc) { /* XXX what if it's running on another cpu?? */ /* * Adjust for the current time slice. This is actually fairly * important since the error here is on the order of a time @@ -510,8 +510,10 @@ calcru(p, up, sp, ip) } totusec = (quad_t)sec * 1000000 + usec; if (totusec < 0) { +#ifndef SMP /* sigh, microtime and fork/exit madness here */ /* XXX no %qd in kernel. Truncate. */ printf("calcru: negative time: %ld usec\n", (long)totusec); +#endif totusec = 0; } u = totusec; diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c index c4922d0afadc..e4459594436a 100644 --- a/sys/kern/kern_shutdown.c +++ b/sys/kern/kern_shutdown.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * @(#)kern_shutdown.c 8.3 (Berkeley) 1/21/94 - * $Id$ + * $Id: kern_shutdown.c,v 1.13 1997/02/22 09:39:10 peter Exp $ */ #include "opt_ddb.h" @@ -166,6 +166,27 @@ boot(howto) { sle_p ep; +#ifdef SMP + int c, spins; + + /* don't accidently start it */ + if (smp_active) { + smp_active = 1; + + spins = 100; + + printf("boot() called on cpu#%d\n", cpunumber()); + while ((c = cpunumber()) != 0) { + if (spins-- < 1) { + printf("timeout waiting for cpu #0!\n"); + break; + } + printf("oops, I'm on cpu#%d, I need to be on cpu#0!\n", + c); + tsleep((caddr_t)&smp_active, PZERO, "cpu0wt", 10); + } + } +#endif ep = shutdown_list1; while (ep) { shutdown_list1 = ep->next; @@ -355,7 +376,11 @@ panic(const char *fmt, ...) else panicstr = fmt; +#ifdef SMP + printf("panic (cpu#%d): ", cpunumber()); +#else printf("panic: "); +#endif va_start(ap, fmt); vprintf(fmt, ap); va_end(ap); diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 04339cde1f9b..f684c0506c4a 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -36,10 +36,11 @@ * SUCH DAMAGE. * * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95 - * $Id: kern_synch.c,v 1.29 1997/02/22 09:39:12 peter Exp $ + * $Id: kern_synch.c,v 1.30 1997/02/27 18:03:48 bde Exp $ */ #include "opt_ktrace.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -176,7 +177,7 @@ schedcpu(arg) { register fixpt_t loadfac = loadfactor(averunnable.ldavg[0]); register struct proc *p; - register int s; + register int s, i, j; register unsigned int newcpu; wakeup((caddr_t)&lbolt); @@ -215,7 +216,16 @@ schedcpu(arg) resetpriority(p); if (p->p_priority >= PUSER) { #define PPQ (128 / NQS) /* priorities per queue */ +#ifdef SMP + for (j = i = 0; i < NCPU; i++) { + if (p == SMPcurproc[i]) + j++; + } + if (!j && + +#else if ((p != curproc) && +#endif p->p_stat == SRUN && (p->p_flag & P_INMEM) && (p->p_priority / PPQ) != (p->p_usrpri / PPQ)) { @@ -226,6 +236,7 @@ schedcpu(arg) p->p_priority = p->p_usrpri; } splx(s); + not_mine: } vmmeter(); timeout(schedcpu, (void *)0, hz); @@ -567,6 +578,10 @@ mi_switch() u -= 1000000; s++; } +#ifdef SMP + if (s < 0) + s = u = 0; +#endif p->p_rtime.tv_usec = u; p->p_rtime.tv_sec = s; diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c index 171ed0e98a45..7cbdac46ea3b 100644 --- a/sys/kern/kern_tc.c +++ b/sys/kern/kern_tc.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * @(#)kern_clock.c 8.5 (Berkeley) 1/21/94 - * $Id: kern_clock.c,v 1.34 1997/03/22 16:52:19 mpp Exp $ + * $Id: kern_clock.c,v 1.35 1997/03/22 20:34:21 mpp Exp $ */ /* Portions of this software are covered by the following: */ @@ -978,7 +978,7 @@ statclock(frame) if (p != NULL) p->p_iticks++; cp_time[CP_INTR]++; - } else if (p != NULL) { + } else if (p != NULL && !(p->p_flag & P_IDLEPROC)) { p->p_sticks++; cp_time[CP_SYS]++; } else diff --git a/sys/kern/kern_timeout.c b/sys/kern/kern_timeout.c index 171ed0e98a45..7cbdac46ea3b 100644 --- a/sys/kern/kern_timeout.c +++ b/sys/kern/kern_timeout.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * @(#)kern_clock.c 8.5 (Berkeley) 1/21/94 - * $Id: kern_clock.c,v 1.34 1997/03/22 16:52:19 mpp Exp $ + * $Id: kern_clock.c,v 1.35 1997/03/22 20:34:21 mpp Exp $ */ /* Portions of this software are covered by the following: */ @@ -978,7 +978,7 @@ statclock(frame) if (p != NULL) p->p_iticks++; cp_time[CP_INTR]++; - } else if (p != NULL) { + } else if (p != NULL && !(p->p_flag & P_IDLEPROC)) { p->p_sticks++; cp_time[CP_SYS]++; } else diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c new file mode 100644 index 000000000000..07fca34efe56 --- /dev/null +++ b/sys/kern/subr_smp.c @@ -0,0 +1,1646 @@ +/* + * Copyright (c) 1996, by Steve Passe + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. The name of the developer may NOT be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $Id: mp_machdep.c,v 1.47 1997/04/26 08:11:49 peter Exp $ + */ + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" +#if defined(APIC_IO) +#if !defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#else /* APIC_IO */ +#if defined(SMP_INVLTLB) +#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER +#endif +#endif /* APIC_IO */ +#define FIX_MP_TABLE_WORKS_NOT + +#include "opt_serial.h" + +#include <sys/param.h> /* for KERNBASE */ +#include <sys/types.h> +#include <sys/sysproto.h> +#include <sys/time.h> +#include <sys/systm.h> + +#include <vm/vm.h> /* for KERNBASE */ +#include <vm/vm_param.h> /* for KERNBASE */ +#include <vm/pmap.h> /* for KERNBASE */ +#include <machine/pmap.h> /* for KERNBASE */ + +#include <machine/smp.h> +#include <machine/apic.h> +#include <machine/mpapic.h> +#include <machine/cpufunc.h> +#include <machine/segments.h> +#include <machine/smptests.h> /** TEST_UPPERPRIO, TEST_DEFAULT_CONFIG */ + +#include <i386/i386/cons.h> /* cngetc() */ + +#if defined(IPI_INTS) +#include <i386/isa/isa_device.h> +#include "vector.h" +#endif /* IPI_INTS */ + +#if defined(SMP_INVLTLB) +#include <i386/isa/icu.h> +#endif /* SMP_INVLTLB */ + +#define WARMBOOT_TARGET 0 +#define WARMBOOT_OFF (KERNBASE + 0x0467) +#define WARMBOOT_SEG (KERNBASE + 0x0469) + +#define BIOS_BASE (0xf0000) +#define BIOS_SIZE (0x10000) +#define BIOS_COUNT (BIOS_SIZE/4) + +#define CMOS_REG (0x70) +#define CMOS_DATA (0x71) +#define BIOS_RESET (0x0f) +#define BIOS_WARM (0x0a) + +/* + * this code MUST be enabled here and in mpboot.s. + * it follows the very early stages of AP boot by placing values in CMOS ram. + * it NORMALLY will never be needed and thus the primitive method for enabling. + * +#define CHECK_POINTS + */ + +#if defined(CHECK_POINTS) +#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) +#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) + +#define CHECK_INIT(D); \ + CHECK_WRITE(0x34, (D)); \ + CHECK_WRITE(0x35, (D)); \ + CHECK_WRITE(0x36, (D)); \ + CHECK_WRITE(0x37, (D)); \ + CHECK_WRITE(0x38, (D)); \ + CHECK_WRITE(0x39, (D)); + +#define CHECK_PRINT(S); \ + printf("%s: %d, %d, %d, %d, %d, %d\n", \ + (S), \ + CHECK_READ(0x34), \ + CHECK_READ(0x35), \ + CHECK_READ(0x36), \ + CHECK_READ(0x37), \ + CHECK_READ(0x38), \ + CHECK_READ(0x39)); + +#else /* CHECK_POINTS */ + +#define CHECK_INIT(D) +#define CHECK_PRINT(S) + +#endif /* CHECK_POINTS */ + + +/** FIXME: what system files declare these??? */ +extern struct region_descriptor r_gdt, r_idt; + +/* global data */ +struct proc *SMPcurproc[NCPU]; +struct pcb *SMPcurpcb[NCPU]; +struct timeval SMPruntime[NCPU]; + +int mp_ncpus; /* # of CPUs, including BSP */ +int mp_naps; /* # of Applications processors */ +int mp_nbusses; /* # of busses */ +int mp_napics; /* # of IO APICs */ +int mpenabled; +int boot_cpu_id; /* designated BSP */ +vm_offset_t cpu_apic_address; +vm_offset_t io_apic_address[NAPIC]; + +u_int32_t cpu_apic_versions[NCPU]; +u_int32_t io_apic_versions[NAPIC]; + +/* + * APIC ID logical/physical mapping structures + */ +int cpu_num_to_apic_id[NCPU]; +int io_num_to_apic_id[NAPIC]; +int apic_id_to_logical[NAPICID]; + +/* + * look for MP compliant motherboard. + */ + +static u_int boot_address; +static u_int base_memory; + +static int picmode; /* 0: virtual wire mode, 1: PIC mode */ +static u_int mpfps; +static int search_for_sig(u_int32_t target, int count); +static int mp_probe(u_int base_top); +static void mp_enable(u_int boot_addr); + +#if defined(IPI_INTS) +static void ipi_intr0(void); +static void ipi_intr1(void); +static void ipi_intr2(void); +static void ipi_intr3(void); +static int +ipi_ihandler_attach(int irq, inthand2_t * func, + unsigned *maskptr, int unit); +#endif /* IPI_INTS */ + + +/* + * calculate usable address in base memory for AP trampoline code + */ +u_int +mp_bootaddress(u_int basemem) +{ + base_memory = basemem * 1024; /* convert to bytes */ + + boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ + if ((base_memory - boot_address) < bootMP_size) + boot_address -= 4096; /* not enough, lower by 4k */ + + return boot_address; +} + + +/* + * startup the SMP processors + */ +void +mp_start(void) +{ + /* look for MP capable motherboard */ + if (mp_probe(base_memory)) + mp_enable(boot_address); + else { + printf("MP FPS NOT FOUND, suggest use of 'mptable' program\n"); + panic("can't continue!\n"); + } + + /* finish pmap initialization - turn off V==P mapping at zero */ + pmap_bootstrap2(); +} + + +/* + * print various information about the SMP system hardware and setup + */ +void +mp_announce(void) +{ + int x; + + printf("FreeBSD/SMP: Multiprocessor motherboard\n"); + printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); + printf(", version: 0x%08x\n", cpu_apic_versions[0]); + for (x = 1; x <= mp_naps; ++x) { + printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); + printf(", version: 0x%08x\n", cpu_apic_versions[x]); + } + +#if defined(APIC_IO) + for (x = 0; x < mp_napics; ++x) { + printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); + printf(", version: 0x%08x\n", io_apic_versions[x]); + } +#else + printf(" Warning: APIC I/O disabled\n"); +#endif /* APIC_IO */ +} + + +/* + * AP cpu's call this to sync up protected mode. + */ +void +init_secondary(void) +{ + int gsel_tss, slot; + + r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; + r_gdt.rd_base = (int) gdt; + lgdt(&r_gdt); /* does magic intra-segment return */ + lidt(&r_idt); + lldt(_default_ldt); + + slot = NGDT + cpunumber(); + gsel_tss = GSEL(slot, SEL_KPL); + gdt[slot].sd.sd_type = SDT_SYS386TSS; + ltr(gsel_tss); + + load_cr0(0x8005003b); /* XXX! */ +} + + +#if defined(APIC_IO) +void +configure_local_apic(void) +{ + u_char byte; + u_int32_t temp; + + if (picmode) { + outb(0x22, 0x70); /* select IMCR */ + byte = inb(0x23); /* current contents */ + byte |= 0x01; /* mask external INTR */ + outb(0x23, byte); /* disconnect 8259s/NMI */ + } + /* mask the LVT1 */ + temp = apic_base[APIC_LVT1]; + temp |= APIC_LVT_M; + apic_base[APIC_LVT1] = temp; +} +#endif /* APIC_IO */ + + +/******************************************************************* + * local functions and data + */ + +static int +mp_probe(u_int base_top) +{ + int x; + u_long segment; + u_int32_t target; + + /* see if EBDA exists */ + if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { + /* search first 1K of EBDA */ + target = (u_int32_t) (segment << 4); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } else { + /*last 1K of base memory, effective 'top of base' is passed in*/ + target = (u_int32_t) (base_top - 0x400); + if ((x = search_for_sig(target, 1024 / 4)) >= 0) + goto found; + } + + /* search the BIOS */ + target = (u_int32_t) BIOS_BASE; + if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) + goto found; + + /* nothing found */ + mpfps = mpenabled = 0; + return 0; + +found: /* please forgive the 'goto'! */ + /* flag fact that we are running multiple processors */ + mpfps = x; + mpenabled = 1; + return 1; +} + + +/* + * start the SMP system + */ +static int parse_mp_table(void); +static void default_mp_table(int type); +static int start_all_aps(u_int boot_addr); + +#if defined(XFAST_IPI32) +#include <machine/md_var.h> +#include <i386/isa/isa_device.h> +extern void Xfastipi32(u_int, u_int, u_int, u_int); +#endif /* XFAST_IPI32 */ + +static void +mp_enable(u_int boot_addr) +{ + int x; +#if defined(APIC_IO) + int apic; + u_int ux; +#if defined(TEST_UPPERPRIO) + u_char select; /* the select register is 8 bits */ + u_int32_t flags; /* the window register is 32 bits */ +#endif /* TEST_UPPERPRIO */ +#endif /* APIC_IO */ + + /* examine the MP table for needed info */ + x = parse_mp_table(); + + /* create pages for (address common) cpu APIC and each IO APIC */ + pmap_bootstrap_apics(); + + /* can't process default configs till the CPU APIC is pmapped */ + if (x) + default_mp_table(x); + +#if defined(APIC_IO) + /* fill the LOGICAL io_apic_versions table */ + for (apic = 0; apic < mp_napics; ++apic) { + ux = io_apic_read(apic, IOAPIC_VER); + io_apic_versions[apic] = ux; + } + + /* + */ + for (apic = 0; apic < mp_napics; ++apic) + if (io_apic_setup(apic) < 0) + panic("IO APIC setup failure\n"); + +#if defined(IPI_INTS) + /* setup IPI INTerrupt mechanism */ + ipi_ihandler_attach( /* irq */ 24, + /* XXX */ (inthand2_t *) ipi_intr0, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 25, + /* XXX */ (inthand2_t *) ipi_intr1, NULL, /* unit */ 0); + ipi_ihandler_attach( /* irq */ 26, + /* XXX */ (inthand2_t *) ipi_intr2, NULL, /* unit */ 0); +#if !defined(SMP_INVLTLB) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); +#else +#if defined(XFAST_IPI32) + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0); + setidt(ICU_OFFSET + 32, + Xfastipi32, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#else + ipi_ihandler_attach( /* irq */ 27, + /* XXX */ (inthand2_t *) ipi_invltlb, NULL, /* unit */ 0); +#endif /* XFAST_IPI32 */ +#endif + +#endif /* IPI_INTS */ + +#if defined(TEST_UPPERPRIO) + +#if 1 + printf("special IRQ10\n"); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~0xff; /** clear vector */ + flags |= 64; + io_apic_write(0, select, flags); +#else + printf("special IRQ10\n"); + cngetc(); + select = IOAPIC_REDTBL10; /** HARD_VECTORXXX: */ + flags = io_apic_read(0, select); + flags &= ~IOART_DELMOD; /* FIXED mode */ + io_apic_write(0, select, flags); + io_apic_write(0, select + 1, boot_cpu_id << 24); +#endif /** 0/1 */ + +#endif /* TEST_UPPERPRIO */ + +#endif /* APIC_IO */ + + /* start each Application Processor */ + start_all_aps(boot_addr); +} + + +/* + * look for the MP spec signature + */ + +/* string defined by the Intel MP Spec as identifying the MP table */ +#define MP_SIG 0x5f504d5f /* _MP_ */ +#define NEXT(X) ((X) += 4) +static int +search_for_sig(u_int32_t target, int count) +{ + int x; + u_int32_t *addr = (u_int32_t *) (KERNBASE + target); + + for (x = 0; x < count; NEXT(x)) + if (addr[x] == MP_SIG) + /* make array index a byte index */ + return (target + (x * sizeof(u_int32_t))); + + return -1; +} + + +#define PROCENTRY_FLAG_EN 0x01 +#define PROCENTRY_FLAG_BP 0x02 +#define IOAPICENTRY_FLAG_EN 0x01 + +/* MP Floating Pointer Structure */ +typedef struct MPFPS { + char signature[4]; + void *pap; + u_char length; + u_char spec_rev; + u_char checksum; + u_char mpfb1; + u_char mpfb2; + u_char mpfb3; + u_char mpfb4; + u_char mpfb5; +} *mpfps_t; +/* MP Configuration Table Header */ +typedef struct MPCTH { + char signature[4]; + u_short base_table_length; + u_char spec_rev; + u_char checksum; + u_char oem_id[8]; + u_char product_id[12]; + void *oem_table_pointer; + u_short oem_table_size; + u_short entry_count; + void *apic_address; + u_short extended_table_length; + u_char extended_table_checksum; + u_char reserved; +} *mpcth_t; + + +typedef struct PROCENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char cpu_flags; + u_long cpu_signature; + u_long feature_flags; + u_long reserved1; + u_long reserved2; +} *proc_entry_ptr; + +typedef struct BUSENTRY { + u_char type; + u_char bus_id; + char bus_type[6]; +} *bus_entry_ptr; + +typedef struct IOAPICENTRY { + u_char type; + u_char apic_id; + u_char apic_version; + u_char apic_flags; + void *apic_address; +} *io_apic_entry_ptr; + +typedef struct INTENTRY { + u_char type; + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} *int_entry_ptr; +/* descriptions of MP basetable entries */ +typedef struct BASETABLE_ENTRY { + u_char type; + u_char length; + char name[16]; +} basetable_entry; + +static basetable_entry basetable_entry_types[] = +{ + {0, 20, "Processor"}, + {1, 8, "Bus"}, + {2, 8, "I/O APIC"}, + {3, 8, "I/O INT"}, + {4, 8, "Local INT"} +}; + +typedef struct BUSDATA { + u_char bus_id; + enum busTypes bus_type; +} bus_datum; + +typedef struct INTDATA { + u_char int_type; + u_short int_flags; + u_char src_bus_id; + u_char src_bus_irq; + u_char dst_apic_id; + u_char dst_apic_int; +} io_int, local_int; + +typedef struct BUSTYPENAME { + u_char type; + char name[7]; +} bus_type_name; + +static bus_type_name bus_type_table[] = +{ + {CBUS, "CBUS"}, + {CBUSII, "CBUSII"}, + {EISA, "EISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {ISA, "ISA"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {PCI, "PCI"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {UNKNOWN_BUSTYPE, "---"}, + {XPRESS, "XPRESS"}, + {UNKNOWN_BUSTYPE, "---"} +}; +/* from MP spec v1.4, table 5-1 */ +static int default_data[7][5] = +{ +/* nbus, id0, type0, id1, type1 */ + {1, 0, ISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {1, 0, EISA, 255, 255}, + {0, 255, 255, 255, 255},/* MCA not supported */ + {2, 0, ISA, 1, PCI}, + {2, 0, EISA, 1, PCI}, + {0, 255, 255, 255, 255} /* MCA not supported */ +}; + + +/* the bus data */ +bus_datum bus_data[NBUS]; + +/* the IO INT data, one entry per possible APIC INTerrupt */ +io_int io_apic_ints[NINTR]; + +static int nintrs; + +#if defined(FIX_MP_TABLE_WORKS) +static void fix_mp_table __P((void)); +#endif /* FIX_MP_TABLE_WORKS */ + +static void processor_entry __P((proc_entry_ptr entry, int *cpu)); +static void io_apic_entry __P((io_apic_entry_ptr entry, int *apic)); +static void bus_entry __P((bus_entry_ptr entry, int *bus)); +static void int_entry __P((int_entry_ptr entry, int *intr)); +static int lookup_bus_type __P((char *name)); + + +/* + * parse an Intel MP specification table + */ +static int +parse_mp_table(void) +{ + int x; + mpfps_t fps; + mpcth_t cth; + int totalSize; + void *position; + int count; + int type; + int apic, bus, cpu, intr; + + /* clear physical APIC ID to logical CPU/IO table */ + for (x = 0; x < NAPICID; ++x) + ID_TO_IO(x) = -1; + + /* clear logical CPU to APIC ID table */ + for (x = 0; x < NCPU; ++x) + CPU_TO_ID(x) = -1; + + /* clear logical IO to APIC ID table */ + for (x = 0; x < NAPIC; ++x) + IO_TO_ID(x) = -1; + + /* clear IO APIC address table */ + for (x = 0; x < NAPIC; ++x) + io_apic_address[x] = ~0; + + /* clear bus data table */ + for (x = 0; x < NBUS; ++x) + bus_data[x].bus_id = 0xff; + + /* clear IO APIC INT table */ + for (x = 0; x < NINTR; ++x) + io_apic_ints[x].int_type = 0xff; + nintrs = 0; + + /* count the BSP */ + mp_ncpus = 1; + + /* setup the cpu/apic mapping arrays */ + boot_cpu_id = -1; + + /* local pointer */ + fps = (mpfps_t) mpfps; + + /* record whether PIC or virtual-wire mode */ + picmode = (fps->mpfb2 & 0x80) ? 1 : 0; + + /* check for use of 'default' configuration */ +#if defined(TEST_DEFAULT_CONFIG) + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return TEST_DEFAULT_CONFIG; +#else + if (fps->mpfb1 != 0) { + /* use default addresses */ + cpu_apic_address = DEFAULT_APIC_BASE; + io_apic_address[0] = DEFAULT_IO_APIC_BASE; + + /* return default configuration type */ + return fps->mpfb1; + } +#endif /* TEST_DEFAULT_CONFIG */ + + if ((cth = fps->pap) == 0) + panic("MP Configuration Table Header MISSING!\n"); + + cpu_apic_address = (vm_offset_t) cth->apic_address; + + totalSize = cth->base_table_length - sizeof(struct MPCTH); + position = (u_char *) cth + sizeof(struct MPCTH); + count = cth->entry_count; + + apic = 0; /* logical apic# start @ 0 */ + bus = 0; /* logical bus# start @ 0 */ + cpu = 1; /* logical cpu# start @ 0, BUT reserve 0 for */ + /* BSP */ + intr = 0; /* unknown */ + + /* walk the table, recording info of interest */ + while (count--) { + switch (type = *(u_char *) position) { + case 0: + processor_entry(position, &cpu); + break; + case 1: + bus_entry(position, &bus); + break; + case 2: + io_apic_entry(position, &apic); + break; + case 3: + int_entry(position, &intr); + break; + case 4: + /* int_entry(position); */ + break; + default: + panic("mpfps Base Table HOSED!\n"); + /* NOTREACHED */ + } + + totalSize -= basetable_entry_types[type].length; + (u_char *) position += basetable_entry_types[type].length; + } + + if (boot_cpu_id == -1) + panic("NO BSP found!\n"); + + /* record # of APs found */ + mp_naps = (cpu - 1); + + /* record # of busses found */ + mp_nbusses = bus; + + /* record # of IO APICs found */ + mp_napics = apic; + + /* record # of IO APICs found */ + nintrs = intr; + +#if defined(FIX_MP_TABLE_WORKS) + /* post scan cleanup */ + fix_mp_table(); +#endif /* FIX_MP_TABLE_WORKS */ + + /* report fact that its NOT a default configuration */ + return 0; +} + + +/* + * parse an Intel MP specification table + */ +#if defined(FIX_MP_TABLE_WORKS) +static void +fix_mp_table(void) +{ + int x; + int y; + int num_pci_bus; + bus_datum bus_record; + + /* + * Fix mis-numbering of the PCI bus and its INT entries if the BIOS + * did it wrong. The MP spec says that when more than 1 PCI bus + * exists the BIOS must begin with bus entries for the PCI bus and use + * actual PCI bus numbering. This implies that when only 1 PCI bus + * exists the BIOS can choose to ignore this ordering, and indeed many + * MP motherboards do ignore it. This causes a problem when the PCI + * sub-system makes requests of the MP sub-system based on PCI bus + * numbers. So here we look for the situation and renumber the + * busses and associated INTs in an effort to "make it right". + */ + + /* count the number of PCI busses */ + for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + ++num_pci_bus; + } + + /* check the 1 PCI bus case for sanity */ + if (num_pci_bus == 1) { + + /* if its in the first slot all is well */ + if (bus_data[0].bus_type == PCI) + return; + + /* mis-numbered, swap with whichever bus uses slot 0 */ + + /* locate the entry holding the PCI bus */ + for (x = 1; x < mp_nbusses; ++x) { + if (bus_data[x].bus_type == PCI) + break; + } + + /* swap the bus entry records */ + bus_record = bus_data[0]; + bus_data[0] = bus_data[x]; + bus_data[x] = bus_record; + + /* swap each relavant INTerrupt entry */ + for (y = 0; y < nintrs; ++y) { + if (io_apic_ints[y].src_bus_id == x) + io_apic_ints[y].src_bus_id = 0; + else + if (io_apic_ints[y].src_bus_id == 0) + io_apic_ints[y].src_bus_id = x; + } + } + /* sanity check if more than 1 PCI bus */ + else + if (num_pci_bus > 1) { + for (x = 0; x < num_pci_bus; ++x) { + if (bus_data[x].bus_type != PCI) { + printf("bad PCI bus numbering\n"); + panic("\n"); + } + } + } +} +#endif /* FIX_MP_TABLE_WORKS */ + + +static void +processor_entry(proc_entry_ptr entry, int *cpu) +{ + int x = *cpu; + + /* check for usability */ + if (!(entry->cpu_flags & PROCENTRY_FLAG_EN)) + return; + + /* check for BSP flag */ + if (entry->cpu_flags & PROCENTRY_FLAG_BP) { + /* always give boot CPU the logical value of 0 */ + x = 0; + boot_cpu_id = entry->apic_id; + } else { + /* add another AP to list, if less than max number of CPUs */ + if (x == NCPU) { + printf("Warning: only using %d of the available CPUs!\n", x); + return; + } + ++(*cpu); + } + + CPU_TO_ID(x) = entry->apic_id; + ID_TO_CPU(entry->apic_id) = x; +} + + +static void +bus_entry(bus_entry_ptr entry, int *bus) +{ + int x, y; + char name[8]; + char c; + + if ((x = (*bus)++) == NBUS) + panic("too many busses, increase 'NBUS'\n"); + + /* encode the name into an index */ + for (y = 0; y < 6; ++y) { + if ((c = entry->bus_type[y]) == ' ') + break; + name[y] = c; + } + name[y] = '\0'; + + if ((y = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) + panic("unknown bus type: '%s'\n", name); + + bus_data[x].bus_id = entry->bus_id; + bus_data[x].bus_type = y; +} + + +static void +io_apic_entry(io_apic_entry_ptr entry, int *apic) +{ + int x; + + if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) + return; + + if ((x = (*apic)++) == NAPIC) + panic("too many APICs, increase 'NAPIC'\n"); + + IO_TO_ID(x) = entry->apic_id; + ID_TO_IO(entry->apic_id) = x; + + io_apic_address[x] = (vm_offset_t) entry->apic_address; +} + + +static int +lookup_bus_type(char *name) +{ + int x; + + for (x = 0; x < MAX_BUSTYPE; ++x) + if (strcmp(bus_type_table[x].name, name) == 0) + return bus_type_table[x].type; + + return UNKNOWN_BUSTYPE; +} + + +static void +int_entry(int_entry_ptr entry, int *intr) +{ + int x; + + if ((x = (*intr)++) == NINTR) + panic("too many INTs, increase 'NINTR'\n"); + + io_apic_ints[x].int_type = entry->int_type; + io_apic_ints[x].int_flags = entry->int_flags; + io_apic_ints[x].src_bus_id = entry->src_bus_id; + io_apic_ints[x].src_bus_irq = entry->src_bus_irq; + io_apic_ints[x].dst_apic_id = entry->dst_apic_id; + io_apic_ints[x].dst_apic_int = entry->dst_apic_int; +} + + +static int +apic_int_is_bus_type(int intr, int bus_type) +{ + int bus; + + for (bus = 0; bus < mp_nbusses; ++bus) + if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) + && ((int) bus_data[bus].bus_type == bus_type)) + return 1; + + return 0; +} + + +/* + * determine which APIC pin an ISA INT is attached to. + */ +#define INTTYPE(I) (io_apic_ints[(I)].int_type) +#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) + +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_isa_apic_irq(int isaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (isaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == isaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, ISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin an EISA INT is attached to. + */ +#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) +int +get_eisa_apic_irq(int eisaIRQ) +{ + int intr; + +#if defined(SMP_TIMER_NC) + if (eisaIRQ == 0) + return -1; +#endif /* SMP_TIMER_NC */ + + for (intr = 0; intr < nintrs; ++intr) /* search each INT record */ + if ((INTTYPE(intr) == 0) + && (SRCBUSIRQ(intr) == eisaIRQ)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, EISA)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSIRQ + + +/* + * determine which APIC pin a PCI INT is attached to. + */ +#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) +#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) +#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) +int +get_pci_apic_irq(int pciBus, int pciDevice, int pciInt) +{ + int intr; + + --pciInt; /* zero based */ + + for (intr = 0; intr < nintrs; ++intr) /* search each record */ + if ((INTTYPE(intr) == 0) +#if defined(FIX_MP_TABLE_WORKS) + && (SRCBUSID(intr) == pciBus) +#endif /* FIX_MP_TABLE_WORKS */ + && (SRCBUSDEVICE(intr) == pciDevice) + && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ + if (apic_int_is_bus_type(intr, PCI)) /* check bus match */ + return INTPIN(intr); /* exact match */ + + return -1; /* NOT found */ +} +#undef SRCBUSLINE +#undef SRCBUSDEVICE +#undef SRCBUSID + +#undef INTPIN +#undef INTTYPE + + +int +undirect_pci_irq(int rirq) +{ +#if defined(READY) + printf("Freeing irq %d for ISA cards.\n", rirq); + /** FIXME: tickle the MB redirector chip */ + return ???; +#else + printf("Freeing (NOT implimented) irq %d for ISA cards.\n", rirq); + return 0; +#endif /* READY */ +} + + +/* + * given a bus ID, return: + * the bus type if found + * -1 if NOT found + */ +int +apic_bus_type(int id) +{ + int x; + + for (x = 0; x < mp_nbusses; ++x) + if (bus_data[x].bus_id == id) + return bus_data[x].bus_type; + + return -1; +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus ID if found + * -1 if NOT found + */ +int +apic_src_bus_id(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_id); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated src bus IRQ if found + * -1 if NOT found + */ +int +apic_src_bus_irq(int apic, int pin) +{ + int x; + + for (x = 0; x < nintrs; x++) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].src_bus_irq); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated INTerrupt type if found + * -1 if NOT found + */ +int +apic_int_type(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_type); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated trigger mode if found + * -1 if NOT found + */ +int +apic_trigger(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return ((io_apic_ints[x].int_flags >> 2) & 0x03); + + return -1; /* NOT found */ +} + + +/* + * given a LOGICAL APIC# and pin#, return: + * the associated 'active' level if found + * -1 if NOT found + */ +int +apic_polarity(int apic, int pin) +{ + int x; + + /* search each of the possible INTerrupt sources */ + for (x = 0; x < nintrs; ++x) + if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && + (pin == io_apic_ints[x].dst_apic_int)) + return (io_apic_ints[x].int_flags & 0x03); + + return -1; /* NOT found */ +} + + +/* + * set data according to MP defaults + * FIXME: probably not complete yet... + */ +static void +default_mp_table(int type) +{ + int ap_cpu_id; +#if defined(APIC_IO) + u_int32_t ux; + int io_apic_id; + int pin; +#endif /* APIC_IO */ + +#if 0 + printf(" MP default config type: %d\n", type); + switch (type) { + case 1: + printf(" bus: ISA, APIC: 82489DX\n"); + break; + case 2: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 3: + printf(" bus: EISA, APIC: 82489DX\n"); + break; + case 4: + printf(" bus: MCA, APIC: 82489DX\n"); + break; + case 5: + printf(" bus: ISA+PCI, APIC: Integrated\n"); + break; + case 6: + printf(" bus: EISA+PCI, APIC: Integrated\n"); + break; + case 7: + printf(" bus: MCA+PCI, APIC: Integrated\n"); + break; + default: + printf(" future type\n"); + break; + /* NOTREACHED */ + } +#endif /* 0 */ + + boot_cpu_id = (apic_base[APIC_ID] & APIC_ID_MASK) >> 24; + ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; + + /* BSP */ + CPU_TO_ID(0) = boot_cpu_id; + ID_TO_CPU(boot_cpu_id) = 0; + + /* one and only AP */ + CPU_TO_ID(1) = ap_cpu_id; + ID_TO_CPU(ap_cpu_id) = 1; + mp_naps = 1; + + /* one and only IO APIC */ +#if defined(APIC_IO) + io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; + + /* + * sanity check, refer to MP spec section 3.6.6, last paragraph + * necessary as some hardware isn't properly setting up the IO APIC + */ +#if defined(REALLY_ANAL_IOAPICID_VALUE) + if (io_apic_id != 2) { +#else + if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { +#endif /* REALLY_ANAL_IOAPICID_VALUE */ + ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ + ux &= ~APIC_ID_MASK; /* clear the ID field */ + ux |= 0x02000000; /* set it to '2' */ + io_apic_write(0, IOAPIC_ID, ux); /* write new value */ + ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ + if ((ux & APIC_ID_MASK) != 0x02000000) + panic("Problem: can't control IO APIC ID, reg: 0x%08x\n", ux); + io_apic_id = 2; + } + IO_TO_ID(0) = io_apic_id; + ID_TO_IO(io_apic_id) = 0; + mp_napics = 1; +#else + mp_napics = 0; +#endif /* APIC_IO */ + + /* fill out bus entries */ + switch (type) { + case 1: + case 2: + case 3: + case 5: + case 6: + mp_nbusses = default_data[type - 1][0]; + bus_data[0].bus_id = default_data[type - 1][1]; + bus_data[0].bus_type = default_data[type - 1][2]; + bus_data[1].bus_id = default_data[type - 1][3]; + bus_data[1].bus_type = default_data[type - 1][4]; + break; + + /* case 4: case 7: MCA NOT supported */ + default: /* illegal/reserved */ + panic("BAD default MP config: %d\n", type); + } + +#if defined(APIC_IO) + /* general cases from MP v1.4, table 5-2 */ + for (pin = 0; pin < 16; ++pin) { + io_apic_ints[pin].int_type = 0; + io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ + io_apic_ints[pin].src_bus_id = 0; + io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ + io_apic_ints[pin].dst_apic_id = io_apic_id; + io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ + } + + /* special cases from MP v1.4, table 5-2 */ + if (type == 2) { + io_apic_ints[2].int_type = 0xff; /* N/C */ + io_apic_ints[13].int_type = 0xff; /* N/C */ +#if !defined(APIC_MIXED_MODE) + /** FIXME: ??? */ + panic("sorry, can't support type 2 default yet\n"); +#endif /* APIC_MIXED_MODE */ + } else + io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ + + if (type == 7) + io_apic_ints[0].int_type = 0xff; /* N/C */ + else + io_apic_ints[0].int_type = 3; /* vectored 8259 */ + + nintrs = 16; +#endif /* APIC_IO */ +} + + +static void install_ap_tramp(u_int boot_addr); +static int start_ap(int logicalCpu, u_int boot_addr); + +/* + * start each AP in our list + */ +static int +start_all_aps(u_int boot_addr) +{ + int x; + u_char mpbiosreason; + u_long mpbioswarmvec; + + /** + * NOTE: this needs further thought: + * where does it get released? + * should it be set to empy? + * + * get the initial mp_lock with a count of 1 for the BSP + */ + mp_lock = (apic_base[APIC_ID] & APIC_ID_MASK) + 1; + + /* initialize BSP's local APIC */ + apic_initialize(1); + + /* install the AP 1st level boot code */ + install_ap_tramp(boot_addr); + + /* save the current value of the warm-start vector */ + mpbioswarmvec = *((u_long *) WARMBOOT_OFF); + outb(CMOS_REG, BIOS_RESET); + mpbiosreason = inb(CMOS_DATA); + + /* start each AP */ + for (x = 1; x <= mp_naps; ++x) { + + /* setup a vector to our boot code */ + *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; + *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ + + /* attempt to start the Application Processor */ + CHECK_INIT(99); /* setup checkpoints */ + if (!start_ap(x, boot_addr)) { + printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); + CHECK_PRINT("trace"); /* show checkpoints */ + /* + * better panic as the AP may be running loose + * somewhere + */ + printf("panic y/n? [n] "); + if (cngetc() != 'n') + panic("bye-bye\n"); + } + CHECK_PRINT("trace"); /* show checkpoints */ + + /* record its version info */ + cpu_apic_versions[x] = cpu_apic_versions[0]; + } + + /* fill in our (BSP) APIC version */ + cpu_apic_versions[0] = apic_base[APIC_VER]; + + /* restore the warmstart vector */ + *(u_long *) WARMBOOT_OFF = mpbioswarmvec; + outb(CMOS_REG, BIOS_RESET); + outb(CMOS_DATA, mpbiosreason); + + /* number of APs actually started */ + return mp_ncpus - 1; +} + + +/* + * load the 1st level AP boot code into base memory. + */ + +/* targets for relocation */ +extern void bigJump(void); +extern void bootCodeSeg(void); +extern void bootDataSeg(void); +extern void MPentry(void); +extern u_int MP_GDT; +extern u_int mp_gdtbase; + +static void +install_ap_tramp(u_int boot_addr) +{ + int x; + int size = *(int *) ((u_long) & bootMP_size); + u_char *src = (u_char *) ((u_long) bootMP); + u_char *dst = (u_char *) boot_addr + KERNBASE; + u_int boot_base = (u_int) bootMP; + u_int8_t *dst8; + u_int16_t *dst16; + u_int32_t *dst32; + + for (x = 0; x < size; ++x) + *dst++ = *src++; + + /* + * modify addresses in code we just moved to basemem. unfortunately we + * need fairly detailed info about mpboot.s for this to work. changes + * to mpboot.s might require changes here. + */ + + /* boot code is located in KERNEL space */ + dst = (u_char *) boot_addr + KERNBASE; + + /* modify the lgdt arg */ + dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); + *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); + + /* modify the ljmp target for MPentry() */ + dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); + *dst32 = ((u_int) MPentry - KERNBASE); + + /* modify the target for boot code segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; + + /* modify the target for boot data segment */ + dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); + dst8 = (u_int8_t *) (dst16 + 1); + *dst16 = (u_int) boot_addr & 0xffff; + *dst8 = ((u_int) boot_addr >> 16) & 0xff; +} + + +/* + * this function starts the AP (application processor) identified + * by the APIC ID 'physicalCpu'. It does quite a "song and dance" + * to accomplish this. This is necessary because of the nuances + * of the different hardware we might encounter. It ain't pretty, + * but it seems to work. + */ +static int +start_ap(int logical_cpu, u_int boot_addr) +{ + int physical_cpu; + int vector; + int cpus; + u_long icr_lo, icr_hi; + + /* get the PHYSICAL APIC ID# */ + physical_cpu = CPU_TO_ID(logical_cpu); + + /* calculate the vector */ + vector = (boot_addr >> 12) & 0xff; + + /* used as a watchpoint to signal AP startup */ + cpus = mp_ncpus; + + /* + * first we do an INIT/RESET IPI this INIT IPI might be run, reseting + * and running the target CPU. OR this INIT IPI might be latched (P5 + * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be + * ignored. + */ + + /* setup the address for the target AP */ + icr_hi = apic_base[APIC_ICR_HI] & ~APIC_ID_MASK; + icr_hi |= (physical_cpu << 24); + apic_base[APIC_ICR_HI] = icr_hi; + + /* do an INIT IPI: assert RESET */ + icr_lo = apic_base[APIC_ICR_LOW] & 0xfff00000; + apic_base[APIC_ICR_LOW] = icr_lo | 0x0000c500; + + /* wait for pending status end */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* do an INIT IPI: deassert RESET */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00008500; + + /* wait for pending status end */ + u_sleep(10000); /* wait ~10mS */ + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + + /* + * next we do a STARTUP IPI: the previous INIT IPI might still be + * latched, (P5 bug) this 1st STARTUP would then terminate + * immediately, and the previously started INIT IPI would continue. OR + * the previous INIT IPI has already run. and this STARTUP IPI will + * run. OR the previous INIT IPI was ignored. and this STARTUP IPI + * will run. + */ + + /* do a STARTUP IPI */ + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* + * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF + * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR + * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is + * recognized after hardware RESET or INIT IPI. + */ + + apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector; + while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK) + /* spin */ ; + u_sleep(200); /* wait ~200uS */ + + /* wait for it to start */ + set_apic_timer(5000000);/* == 5 seconds */ + while (read_apic_timer()) + if (mp_ncpus > cpus) + return 1; /* return SUCCESS */ + + return 0; /* return FAILURE */ +} + + +#if defined(IPI_INTS) + +static void +ipi_intr0(void) +{ + printf("IPI 0\n"); +} + + +static void +ipi_intr1(void) +{ + printf("IPI 1\n"); +} + + +static void +ipi_intr2(void) +{ + printf("IPI 2\n"); +} + + +static void +ipi_intr3(void) +{ + printf("IPI 3\n"); +} + +/*----------------------------------------------------------------------- +** +** Register an interupt handler for an IPI. +** (Stolen from the PCI<->ISA glue code) +** +**----------------------------------------------------------------------- +*/ + +static int +ipi_ihandler_attach(int irq, inthand2_t * func, unsigned *maskptr, int unit) +{ + char buf[16]; + char *cp; + int free_id, id, result; + + sprintf(buf, "ipi irq%d", irq); + for (cp = intrnames, free_id = 0, id = 0; id < NR_DEVICES; id++) { + if (strcmp(cp, buf) == 0) + break; + if (free_id <= 0 && strcmp(cp, "ipi irqnn") == 0) + free_id = id; + while (*cp++ != '\0'); + } + if (id == NR_DEVICES) { + id = free_id; + if (id == 0) { + /* + * All ipi irq counters are in use, perhaps because + * config is old so there aren't any. Abuse the clk0 + * counter. + */ + printf( + "ipi_ihandler_attach: counting ipi irq%d's as clk0 irqs\n", + irq); + } + } + result = register_intr( + irq, /* isa irq */ + id, /* device id */ + 0, /* flags? */ + func, /* handler */ + maskptr, /* mask pointer */ + unit); /* handler arg */ + + if (result) { + printf("WARNING: ipi_ihandler_attach: result=%d\n", result); + return (result); + }; + + return (0); +} +#endif /* IPI_INTS */ + + +#ifdef SMP_INVLTLB +/* + * Flush the TLB on all other CPU's + * + * XXX: Needs to handshake and wait for completion before proceding. + */ + +void +smp_invltlb() +{ + if (smp_active) { + if (invldebug & 2) +#if defined(XFAST_IPI32) + all_but_self_ipi(ICU_OFFSET + 32); +#else + all_but_self_ipi(ICU_OFFSET + 27); +#endif /* XFAST_IPI32 */ + } +} + +void +invlpg(u_int addr) +{ + __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); + smp_invltlb(); +} + +void +invltlb(void) +{ + u_long temp; + /* + * This should be implemented as load_cr3(rcr3()) when load_cr3() is + * inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); + smp_invltlb(); +} +/* + * Handles recieving an "IRQ 27", the invalidate tlb IPI.. + */ +void +ipi_invltlb(void) +{ + u_long temp; + + if (invldebug & 4) { + /* + * This should be implemented as load_cr3(rcr3()) when + * load_cr3() is inlined. + */ + __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) + :: "memory"); + + } +} +#endif /* SMP_INVLTLB */ diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index 8a86965196d5..aa4e3c5d39d5 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)trap.c 7.4 (Berkeley) 5/13/91 - * $Id: trap.c,v 1.91 1997/04/07 07:15:55 peter Exp $ + * $Id: trap.c,v 1.92 1997/04/14 13:52:52 bde Exp $ */ /* @@ -44,6 +44,7 @@ #include "opt_ktrace.h" #include "opt_ddb.h" +#include "opt_smp.h" #include <sys/param.h> #include <sys/systm.h> @@ -76,6 +77,7 @@ #include <machine/reg.h> #include <machine/trap.h> #include <machine/../isa/isa_device.h> +#include <machine/smp.h> #ifdef POWERFAIL_NMI #include <sys/syslog.h> @@ -85,7 +87,11 @@ #include "isa.h" #include "npx.h" +#ifdef SMP +extern struct i386tss *SMPcommon_tss_ptr[NCPU]; +#else extern struct i386tss common_tss; +#endif int (*pmath_emulate) __P((struct trapframe *)); @@ -678,6 +684,9 @@ trap_fatal(frame) printf("\n\nFatal trap %d: %s while in %s mode\n", type, trap_msg[type], ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel"); +#ifdef SMP + printf("cpunumber = %d\n", cpunumber()); +#endif if (type == T_PAGEFLT) { printf("fault virtual address = 0x%x\n", eva); printf("fault code = %s %s, %s\n", @@ -740,6 +749,7 @@ trap_fatal(frame) if (kdb_trap (type, 0, frame)) return; #endif + printf("trap number = %d\n", type); if (type <= MAX_TRAP_MSG) panic(trap_msg[type]); else @@ -761,11 +771,20 @@ trap_fatal(frame) void dblfault_handler() { +#ifdef SMP + int x = cpunumber(); +#endif printf("\nFatal double fault:\n"); +#ifdef SMP + printf("eip = 0x%x\n", SMPcommon_tss_ptr[x]->tss_eip); + printf("esp = 0x%x\n", SMPcommon_tss_ptr[x]->tss_esp); + printf("ebp = 0x%x\n", SMPcommon_tss_ptr[x]->tss_ebp); +#else printf("eip = 0x%x\n", common_tss.tss_eip); printf("esp = 0x%x\n", common_tss.tss_esp); printf("ebp = 0x%x\n", common_tss.tss_ebp); +#endif panic("double fault"); } diff --git a/sys/pci/pci.c b/sys/pci/pci.c index 8c1bc7dbc19b..96a71e841cf0 100644 --- a/sys/pci/pci.c +++ b/sys/pci/pci.c @@ -1,6 +1,6 @@ /************************************************************************** ** -** $Id: pci.c,v 1.68 1997/03/25 19:12:08 se Exp $ +** $Id: pci.c,v 1.69 1997/04/23 19:43:20 se Exp $ ** ** General subroutines for the PCI bus. ** pci_configure () @@ -46,6 +46,8 @@ **======================================================== */ +#include "opt_smp.h" + #include <sys/param.h> #include <sys/systm.h> #include <sys/malloc.h> @@ -70,9 +72,6 @@ #include <pci/pcibus.h> #include <pci/pci_ioctl.h> -#define PCI_MAX_IRQ (16) - - /*======================================================== ** ** Structs and Functions @@ -403,6 +402,10 @@ static void pci_attach (int bus, int dev, int func, u_char reg; u_char pciint; int irq; +#if defined(APIC_IO) + u_char airq = 0xff; + u_char rirq = 0xff; +#endif /* APIC_IO */ pcici_t tag = pcibus->pb_tag (bus, dev, func); /* @@ -437,14 +440,33 @@ static void pci_attach (int bus, int dev, int func, ** and we cannot bind the pci interrupt. */ +#if defined(APIC_IO) + if (irq && (irq != 0xff)) { + airq = get_pci_apic_irq (bus, dev, pciint); + if (airq != 0xff) { /* APIC IRQ exists */ + rirq = irq; /* 're-directed' IRQ */ + irq = airq; /* use APIC IRQ */ + } + printf ("%d", irq); + } +#else if (irq && (irq != 0xff)) printf ("%d", irq); +#endif /* APIC_IO */ else printf ("??"); }; printf (" on pci%d:%d:%d\n", bus, dev, func); +#if defined(APIC_IO) + if (airq != 0xff) { /* APIC IRQ exists */ + data = PCI_INTERRUPT_LINE_INSERT(data, airq); + pci_conf_write (tag, PCI_INTERRUPT_REG, data); + undirect_pci_irq (rirq); /* free for ISA card */ + } +#endif /* APIC_IO */ + /* ** Read the current mapping, ** and update the pcicb fields. diff --git a/sys/pci/pcibus.h b/sys/pci/pcibus.h index 4e68144f6195..60c22b9da990 100644 --- a/sys/pci/pcibus.h +++ b/sys/pci/pcibus.h @@ -1,6 +1,6 @@ /************************************************************************** ** -** $Id$ +** $Id: pcibus.h,v 1.9 1997/02/22 09:44:11 peter Exp $ ** ** Declarations for pci bus driver. ** @@ -72,6 +72,8 @@ **----------------------------------------------------------------- */ +#include "opt_smp.h" + struct pcibus { char *pb_name; void (*pb_setup ) (void); @@ -86,7 +88,11 @@ struct pcibus { int (*pb_imaskexc)(int irq, unsigned *maskptr); }; +#if defined(APIC_IO) +#define PCI_MAX_IRQ (24) +#else #define PCI_MAX_IRQ (16) +#endif /* APIC_IO */ /* ** The following structure should be generated by the driver diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h index c426d81c3c50..d1ac90e62e97 100644 --- a/sys/sys/kernel.h +++ b/sys/sys/kernel.h @@ -39,15 +39,19 @@ * SUCH DAMAGE. * * @(#)kernel.h 8.3 (Berkeley) 1/21/94 - * $Id: kernel.h,v 1.27 1997/04/06 11:14:12 dufault Exp $ + * $Id: kernel.h,v 1.28 1997/04/22 06:55:47 jdp Exp $ */ #ifndef _SYS_KERNEL_H_ #define _SYS_KERNEL_H_ +#ifdef KERNEL + +#include "opt_smp.h" +#include <machine/smp.h> + /* Global variables for the kernel. */ -#ifdef KERNEL /* 1.1 */ extern long hostid; extern char hostname[MAXHOSTNAMELEN]; @@ -59,7 +63,12 @@ extern char kernelname[MAXPATHLEN]; /* 1.2 */ extern volatile struct timeval mono_time; extern struct timeval boottime; +#ifdef SMP +#define runtime (SMPruntime[cpunumber()]) +#else /* !SMP */ extern struct timeval runtime; +#endif /* SMP */ + extern struct timeval time; /* nonvolatile at ipl >= splclock() */ extern struct timezone tz; /* XXX */ @@ -168,6 +177,8 @@ enum sysinit_sub_id { SI_SUB_KTHREAD_PAGE = 0xe4000000, /* pageout daemon*/ SI_SUB_KTHREAD_VM = 0xe8000000, /* vm daemon*/ SI_SUB_KTHREAD_UPDATE = 0xec000000, /* update daemon*/ + SI_SUB_KTHREAD_IDLE = 0xee000000, /* idle procs*/ + SI_SUB_SMP = 0xf0000000, /* idle procs*/ SI_SUB_RUN_SCHEDULER = 0xffffffff /* scheduler: no return*/ }; diff --git a/sys/sys/proc.h b/sys/sys/proc.h index bc65877a9180..31105119c4ad 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * @(#)proc.h 8.15 (Berkeley) 5/19/95 - * $Id: proc.h,v 1.34 1997/04/07 07:16:02 peter Exp $ + * $Id: proc.h,v 1.35 1997/04/07 09:35:15 peter Exp $ */ #ifndef _SYS_PROC_H_ @@ -49,6 +49,11 @@ #include <sys/queue.h> #include <sys/param.h> +#ifdef KERNEL +#include "opt_smp.h" +#include <machine/smp.h> +#endif + /* * One structure allocated per session. */ @@ -138,7 +143,9 @@ struct proc { struct vnode *p_textvp; /* Vnode of executable. */ char p_lock; /* Process lock (prevent swap) count. */ - char p_pad2[3]; /* alignment */ + char p_oncpu; /* Which cpu we are on */ + char p_lastcpu; /* Last cpu we were on */ + char p_pad2; /* alignment */ char *p_selbits; /* For select(), bits */ u_int p_selbits_size; /* For select(), fd_set size (bytes) */ @@ -218,6 +225,7 @@ struct proc { #define P_SWAPPING 0x40000 /* Process is being swapped. */ #define P_SWAPINREQ 0x80000 /* Swapin request due to wakeup */ +#define P_IDLEPROC 0x100000 /* Process is an idle-eater, don't count */ /* * MOVE TO ucred.h? @@ -265,7 +273,12 @@ extern u_long pidhash; extern LIST_HEAD(pgrphashhead, pgrp) *pgrphashtbl; extern u_long pgrphash; +#ifdef SMP +#define curproc (SMPcurproc[cpunumber()]) +#else /* !SMP */ extern struct proc *curproc; /* Current running proc. */ +#endif /* SMP */ + extern struct proc proc0; /* Process slot for swapper. */ extern int nprocs, maxproc; /* Current and max number of procs. */ extern int maxprocperuid; /* Max procs per uid. */ @@ -280,6 +293,8 @@ extern struct prochd qs[]; extern struct prochd rtqs[]; extern struct prochd idqs[]; extern int whichqs; /* Bit mask summary of non-empty Q's. */ +extern int whichrtqs; /* Bit mask summary of non-empty Q's. */ +extern int whichidqs; /* Bit mask summary of non-empty Q's. */ struct prochd { struct proc *ph_link; /* Linked list of running processes. */ struct proc *ph_rlink; diff --git a/sys/sys/smp.h b/sys/sys/smp.h new file mode 100644 index 000000000000..905ca3529327 --- /dev/null +++ b/sys/sys/smp.h @@ -0,0 +1,135 @@ +/* + * ---------------------------------------------------------------------------- + * "THE BEER-WARE LICENSE" (Revision 42): + * <phk@FreeBSD.org> wrote this file. As long as you retain this notice you + * can do whatever you want with this stuff. If we meet some day, and you think + * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp + * ---------------------------------------------------------------------------- + * + * $Id: smp.h,v 1.29 1997/04/26 08:11:50 peter Exp $ + * + */ + +#ifndef _MACHINE_SMP_H_ +#define _MACHINE_SMP_H_ + +#ifdef KERNEL + +#include "opt_smp.h" +#include "opt_smp_invltlb.h" /* aiee! (for cpufunc.h!) */ + +#if defined(SMP) && !defined(NCPU) +# define NCPU 2 +#endif /* SMP && NCPU */ + +#if defined(SMP) || defined(APIC_IO) + +/* global data in mpboot.s */ +extern int bootMP_size; + +/* functions in mpboot.s */ +void bootMP __P((void)); + +/* global data in mplock.s */ +extern u_int mp_lock; + +/* functions in mplock.s */ +void get_mplock __P((void)); +void rel_mplock __P((void)); +void expect_mplock __P((void)); + +/* global data in mp_machdep.c */ +extern struct proc* SMPcurproc[NCPU]; +extern struct pcb* SMPcurpcb[NCPU]; +extern struct timeval SMPruntime[NCPU]; +extern int mp_ncpus; +extern int mp_naps; +extern int mp_nbusses; +extern int mp_napics; +extern int mp_picmode; +extern int mpenabled; +extern int boot_cpu_id; +extern vm_offset_t cpu_apic_address; +extern vm_offset_t io_apic_address[]; +extern u_int32_t cpu_apic_versions[]; +extern u_int32_t io_apic_versions[]; +extern int cpu_num_to_apic_id[]; +extern int io_num_to_apic_id[]; +extern int apic_id_to_logical[]; + +/* functions in mp_machdep.c */ +u_int mp_bootaddress __P((u_int)); +void mp_start __P((void)); +void mp_announce __P((void)); +int get_isa_apic_irq __P((int)); +int get_eisa_apic_irq __P((int)); +int get_pci_apic_irq __P((int, int, int)); +int undirect_pci_irq __P((int)); +int apic_bus_type __P((int)); +int apic_src_bus_id __P((int, int)); +int apic_src_bus_irq __P((int, int)); +int apic_int_type __P((int, int)); +int apic_trigger __P((int, int)); +int apic_polarity __P((int, int)); +void configure_local_apic __P((void)); +void init_secondary __P((void)); +#ifdef SMP_INVLTLB +void ipi_invltlb __P((void)); +void smp_invltlb __P((void)); +#endif + +/* global data in mpapic.c */ +extern volatile u_int* apic_base; + +#if defined(MULTIPLE_IOAPICS) +#error MULTIPLE_IOAPICSXXX +#else +extern volatile u_int* io_apic_base; +#endif /* MULTIPLE_IOAPICS */ + +/* functions in mpapic.c */ +void apic_initialize __P((int)); +int selected_apic_ipi __P((u_int, int, int)); +int io_apic_setup __P((int)); +int ext_int_setup __P((int, int)); +void write_io_apic_mask24 __P((int, u_int32_t)); + +#if defined(READY) +void clr_io_apic_mask24 __P((int, u_int32_t)); +void set_io_apic_mask24 __P((int, u_int32_t)); +#endif /* READY */ + +void set_apic_timer __P((int)); +int read_apic_timer __P((void)); +void u_sleep __P((int)); + +/* global data in init_smp.c */ +extern int smp_active; +extern int invldebug; + +/* in pmap.c FIXME: belongs in pmap.h??? */ +void pmap_bootstrap_apics __P((void)); +void pmap_bootstrap2 __P((void)); + +#if 0 +/* chicken and egg problem... */ +static __inline unsigned +cpunumber(void) +{ + return (unsigned)ID_TO_CPU((apic_base[APIC_ID] & APIC_ID_MASK) >> 24); +} +#else +/* + * we 'borrow' this info from apic.h + * this will go away soon... + */ +static __inline unsigned +cpunumber(void) +{ + return (unsigned)(apic_id_to_logical[(apic_base[8] & 0x0f000000) >> 24]); +} +#endif /* 0 */ + +#endif /* SMP || APIC_IO */ +#endif /* KERNEL */ +#endif /* _MACHINE_SMP_H_ */ diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c index a875230a78e9..af36cfd22566 100644 --- a/sys/vm/vm_kern.c +++ b/sys/vm/vm_kern.c @@ -61,12 +61,14 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_kern.c,v 1.33 1997/02/22 09:48:21 peter Exp $ + * $Id: vm_kern.c,v 1.34 1997/03/31 11:11:24 davidg Exp $ */ /* * Kernel memory management. */ +#include "opt_smp.h" +#include "opt_smp_privpages.h" #include <sys/param.h> #include <sys/systm.h> @@ -99,6 +101,10 @@ vm_map_t mb_map=0; int mb_map_full=0; vm_map_t io_map=0; vm_map_t phys_map=0; +#if defined(SMP) && defined(SMP_PRIVPAGES) +vm_map_t ppage_map=0; +#endif + /* * kmem_alloc_pageable: diff --git a/sys/vm/vm_kern.h b/sys/vm/vm_kern.h index b23150f621a2..5d85318b42c8 100644 --- a/sys/vm/vm_kern.h +++ b/sys/vm/vm_kern.h @@ -61,7 +61,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_kern.h,v 1.12 1997/02/22 09:48:21 peter Exp $ + * $Id: vm_kern.h,v 1.13 1997/03/31 11:11:26 davidg Exp $ */ #ifndef _VM_VM_KERN_H_ @@ -78,6 +78,7 @@ extern vm_map_t clean_map; extern vm_map_t phys_map; extern vm_map_t exec_map; extern vm_map_t u_map; +extern vm_map_t ppage_map; extern vm_offset_t kernel_vm_end; /* XXX - elsewhere? */ diff --git a/sys/vm/vm_meter.c b/sys/vm/vm_meter.c index c58423ce3636..b6ac38cb51a3 100644 --- a/sys/vm/vm_meter.c +++ b/sys/vm/vm_meter.c @@ -31,7 +31,7 @@ * SUCH DAMAGE. * * @(#)vm_meter.c 8.4 (Berkeley) 1/4/94 - * $Id$ + * $Id: vm_meter.c,v 1.20 1997/02/22 09:48:25 peter Exp $ */ #include <sys/param.h> @@ -79,6 +79,8 @@ loadav(struct loadavg *avg) register struct proc *p; for (nrun = 0, p = allproc.lh_first; p != 0; p = p->p_list.le_next) { + if (p->p_flag & P_IDLEPROC) + continue; switch (p->p_stat) { case SSLEEP: if (p->p_priority > PZERO || p->p_slptime != 0) diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index 4de7814a6a6e..ded43748e993 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -61,7 +61,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id$ + * $Id: vm_object.c,v 1.90 1997/02/22 09:48:28 peter Exp $ */ /* @@ -1451,6 +1451,10 @@ vm_object_in_map( object) return 1; if( _vm_object_in_map( u_map, object, 0)) return 1; +#if defined(SMP) && defined(SMP_PRIVPAGES) + if( _vm_object_in_map( ppage_map, object, 0)) + return 1; +#endif return 0; } |